diff --git a/README.md b/README.md index 4ffdc5109..8e0fc01f1 100644 --- a/README.md +++ b/README.md @@ -891,7 +891,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2) ✅ SHA extensions @@ -907,7 +907,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2) ✅ SHA extensions @@ -923,7 +923,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2) ✅ SHA extensions diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml index 4f28100c0..23c04ea14 100644 --- a/docs/algorithms/ml-dsa-44.toml +++ b/docs/algorithms/ml-dsa-44.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa44.rs"] asm = ["mldsa44", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2)" } diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml index 6683e2898..04ce839cb 100644 --- a/docs/algorithms/ml-dsa-65.toml +++ b/docs/algorithms/ml-dsa-65.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa65.rs"] asm = ["mldsa65", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2)" } diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml index 649298bea..88e39d47a 100644 --- a/docs/algorithms/ml-dsa-87.toml +++ b/docs/algorithms/ml-dsa-87.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa87.rs"] asm = ["mldsa87", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2)" } diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean index 5206623ad..56aac4f8e 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean @@ -29,6 +29,7 @@ def prims : Prims where power2Round := Round.power2Round simpleBitPack := Pack.simpleBitPack bitPack := Pack.bitPack + rej4 := Sample.Rej4.rejNTT4 /-- The primitives, with the polynomial arithmetic of `B`. -/ def primsWith (B : Arith.Backend) : Prims := @@ -38,6 +39,7 @@ def primsWith (B : Arith.Backend) : Prims := mul := B.mul mulAdd := B.mulAdd add := B.add + rej4 := B.rej4 sfx := B.sfx } end VG.Impl.MlDsa.X86_64.KeyGen diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean index 4f3d0ef4c..8335353dd 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean @@ -17,18 +17,22 @@ The layout of `scratch` (in bytes): the Keccak state at 0 and the sponge functions' working space at 200, the saved registers at 840 (as ML-KEM's); `k` and `ℓ` at 896 (`oKL`); `(ρ, ρ′, K)` at 1024 (`oHX`, 128 bytes); the seed of `RejNTTPoly` at 1152 (`oSA`, 34 bytes) and of `RejBoundedPoly` at -1216 (`oSB`, 66 bytes); the working space of the primitives at 2048 +1216 (`oSB`, 66 bytes); the four seeds of `vg_mldsa_rej_ntt_poly4` at 1408 +(`oSA4`, 136 bytes); the working space of the primitives at 2048 (2048 bytes); and polynomials of 1024 bytes from 4096 (`oP j`): `Â[r, s]` is polynomial `rℓ + s`, `s₁[j]` (then `ŝ₁[j]`) polynomial `kℓ + j`, `s₂[i]` polynomial `kℓ + ℓ + i`, and `t`, `t₁` and `t₀` the three after -them. +them; then the working space of `vg_mldsa_rej_ntt_poly4` (8 KiB, `oR4`). 1. `(ρ, ρ′, K) = H(ξ ‖ k ‖ ℓ, 128)`, and `ρ` and `ρ′` to the seeds. -2. `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)` and `s₁ ‖ s₂ = RejBoundedPoly(ρ′ ‖ r ‖ 0)` - (`ExpandA`, `ExpandS`). After each, `r15 ← r15 ∧ result`, and the - polynomial is ANDed with `-result` (`mask`): it is zero if the sampler - failed, so that every polynomial is reduced, and small, whatever the - samplers return, without a branch. +2. `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)`, four consecutive entries at a time + (`vg_mldsa_rej_ntt_poly4`, from the seeds at `oSA4`, each `ρ` with its + entry's indices) and the last `kℓ mod 4` one at a time, and + `s₁ ‖ s₂ = RejBoundedPoly(ρ′ ‖ r ‖ 0)` (`ExpandA`, `ExpandS`). After each + call, `r15 ← r15 ∧ result`, and the polynomials it sampled are ANDed with + `-result` (`mask`): they are zero if the sampler failed, so that every + polynomial is reduced, and small, whatever the samplers return, without a + branch. 3. `ρ` and `K` to `sk`; `s₁` and `s₂`, `BitPack`ed, to `sk`; `ŝ₁ = NTT(s₁)`. 4. For each row `i`: `t = NTT⁻¹(Σⱼ Â[i, j] ŝ₁[j]) + s₂[i]`, `Power2Round`, and `t₁` `SimpleBitPack`ed to `pk`, `t₀` `BitPack`ed to `sk`; `ρ` to `pk`. @@ -50,6 +54,7 @@ def oKL : Nat := 896 def oHX : Nat := 1024 def oSA : Nat := 1152 def oSB : Nat := 1216 +def oSA4 : Nat := 1408 /-- Polynomial `j`. -/ def oP (j : Nat) : Nat := 4096 + 1024 * j @@ -60,6 +65,8 @@ abbrev sP (p : Params) (r : Nat) : Ptr := sc (oP (p.k * p.ℓ + r)) abbrev tP (p : Params) : Ptr := sc (oP (p.k * p.ℓ + p.ℓ + p.k)) abbrev t1P (p : Params) : Ptr := sc (oP (p.k * p.ℓ + p.ℓ + p.k + 1)) abbrev t0P (p : Params) : Ptr := sc (oP (p.k * p.ℓ + p.ℓ + p.k + 2)) +/-- The working space of `vg_mldsa_rej_ntt_poly4`. -/ +def oR4 (p : Params) : Nat := oP (p.k * p.ℓ + p.ℓ + p.k + 3) /-- The length of a packed polynomial of `s₁` or `s₂`, `32 · bitlen (2η)`. -/ def lenS (p : Params) : Nat := 32 * bitlen (2 * p.η) @@ -89,6 +96,9 @@ def addAt (sfx : String) (c : Prog isa) (f g : Ptr) : Prog isa := def rejNttAt (c : Prog isa) (seed a : Ptr) : Prog isa := .seq (.block (lea .rdi seed ++ lea .rsi a ++ lea .rdx (sc oSS))) (.call "vg_mldsa_rej_ntt_poly" c) +def rej4At (c : Prog isa) (sfx : String) (a w : Ptr) : Prog isa := + .seq (.block (lea .rdi (sc oSA4) ++ lea .rsi a ++ lea .rdx w)) (.call ("vg_mldsa_rej_ntt_poly4" ++ sfx) c) + def rejBoundedAt (c : Prog isa) (seed : Ptr) (eta : Nat) (a : Ptr) : Prog isa := .seq (.block (lea .rdi seed ++ imm .rsi eta ++ lea .rdx a ++ lea .rcx (sc oSS))) (.call "vg_mldsa_rej_bounded_poly" c) @@ -104,10 +114,11 @@ def bitPackAt (c : Prog isa) (f : Ptr) (a b : Nat) (out : Ptr) (len : Nat) : Pro .seq (.block (lea .rdi f ++ imm .rsi a ++ imm .rdx b ++ lea .rcx out ++ imm .r8 len)) (.call "vg_mldsa_bit_pack" c) -/-- `r15 ← r15 ∧ eax`, and the polynomial at `a` ANDed with `-eax` (`eax` is 0 or 1). -/ -def mask (a : Ptr) : Prog isa := +/-- `r15 ← r15 ∧ eax`, and the polynomial at `a` (or the `N` coefficients from `a`) ANDed with `-eax` +(`eax` is 0 or 1). -/ +def mask (a : Ptr) (N : Nat := 256) : Prog isa := .seq (.block ([.alu32 .and .r15 (.reg .rax), .mov32 .r8 (.imm 0), .alu32 .sub .r8 (.reg .rax)] ++ - lea .rdi a ++ imm .rcx 256)) + lea .rdi a ++ imm .rcx N)) (.loop (.block [.mov32 .rax (.mem (at_ .rdi 0)), .alu32 .and .rax (.reg .r8), .store32 (at_ .rdi 0) .rax, .alu .add .rdi (.imm 4), .alu .sub .rcx (.imm 1)]) .ne) @@ -120,13 +131,28 @@ def pro : List Instr := topPro .rcx [(.rbp, .rdi), (.r12, .rsi), (.r13, .rdx)] def seeds (p : Params) : Prog isa := .seq (.block (setB (sc oKL) p.k ++ setB (sc (oKL + 1)) p.ℓ)) (.seq (hashAt [((.rbp, 0), 32), (sc oKL, 2)] 136 0x1f (sc oHX) 128) - (.seq (copy (sc oSA) (sc oHX) 32) (.seq (copy (sc oSB) (sc (oHX + 32)) 64) (.block (setB (sc (oSB + 65)) 0))))) + (.seq (copy (sc oSA) (sc oHX) 32) (.seq (copy (sc oSB) (sc (oHX + 32)) 64) (.seq (.block (setB (sc (oSB + 65)) 0)) + (.seq (copy (sc oSA4) (sc oHX) 32) (.seq (copy (sc (oSA4 + 34)) (sc oHX) 32) + (.seq (copy (sc (oSA4 + 68)) (sc oHX) 32) (copy (sc (oSA4 + 102)) (sc oHX) 32)))))))) /-- `Â[e / ℓ, e % ℓ] = RejNTTPoly(ρ ‖ e % ℓ ‖ e / ℓ)`. -/ def expA (P : Prims) (p : Params) (e : Nat) : Prog isa := .seq (.block (setB (sc (oSA + 32)) (e % p.ℓ) ++ setB (sc (oSA + 33)) (e / p.ℓ))) (.seq (rejNttAt P.rejNtt (sc oSA) (aP e)) (mask (aP e))) +/-- The indices of entry `e + k` of `Â` to seed `k` of `oSA4`. -/ +def setSR (p : Params) (e k : Nat) : List Instr := + setB (sc (oSA4 + 34 * k + 32)) ((e + k) % p.ℓ) ++ setB (sc (oSA4 + 34 * k + 32 + 1)) ((e + k) / p.ℓ) + +/-- Entries `4g, …, 4g + 3` of `Â`. -/ +def expA4 (P : Prims) (p : Params) (g : Nat) : Prog isa := + .seq (.block (setSR p (4 * g) 0)) (.seq (.block (setSR p (4 * g) 1)) (.seq (.block (setSR p (4 * g) 2)) + (.seq (.block (setSR p (4 * g) 3)) (.seq (rej4At P.rej4 P.sfx (aP (4 * g)) (sc (oR4 p))) (mask (aP (4 * g)) 1024))))) + +/-- The entries of `Â`: four at a time, then the last `kℓ mod 4` one at a time. -/ +def expAll (P : Prims) (p : Params) : Prog isa := + .seq (seqR (expA4 P p) 0 (p.k * p.ℓ / 4)) (seqR (expA P p) (4 * (p.k * p.ℓ / 4)) (p.k * p.ℓ % 4)) + /-- Entry `r` of `s₁ ‖ s₂`: `RejBoundedPoly(ρ′ ‖ r ‖ 0)`. -/ def expS (P : Prims) (p : Params) (r : Nat) : Prog isa := .seq (.block (setB (sc (oSB + 64)) r)) @@ -162,7 +188,7 @@ def rest (P : Prims) (p : Params) : Prog isa := /-- `vg_mldsa*_keygen` for the parameter set `p`, calling the primitives `P`. -/ def keyGen (P : Prims) (p : Params) : Prog isa := - .seq (.block pro) (.seq (seeds p) (.seq (seqR (expA P p) 0 (p.k * p.ℓ)) + .seq (.block pro) (.seq (seeds p) (.seq (expAll P p) (.seq (seqR (expS P p) 0 (p.ℓ + p.k)) (.seq (rest P p) (.block topEpi))))) end VG.Impl.MlDsa.X86_64.KeyGen diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean index a2ed604be..9c33d6845 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean @@ -36,6 +36,8 @@ structure Prims where simpleBitPack : Prog isa /-- `vg_mldsa_bit_pack` -/ bitPack : Prog isa + /-- `vg_mldsa_rej_ntt_poly4` -/ + rej4 : Prog isa /-- What the names of the polynomial arithmetic's functions end with (`Arith.Backend`). -/ sfx : String := "" diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Base.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Base.lean index a8ab5a6ef..54a459c66 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Base.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Base.lean @@ -32,6 +32,13 @@ structure Callee (c : Prog isa) (k : Nat → Contract isa) : Prop where nosp : NoSp c depth : c.depth ≤ 2 +/-- `vg_mldsa_rej_ntt_poly4`: verified with 24 bytes of stack, never writing `rsp` but by calls nested at +most three deep. -/ +structure Callee4 (c : Prog isa) : Prop where + verified : Verified X86_64.target c (Spec.MlDsa.rejNTT4Contract X86_64.abi 24) + nosp : NoSp c + depth : c.depth ≤ 3 + /-- Verified implementations of the primitives key generation calls. -/ structure PrimsOk (P : Prims) : Prop where ntt : Callee P.ntt (fun stk => Spec.MlDsa.nttContract X86_64.abi stk) @@ -44,6 +51,7 @@ structure PrimsOk (P : Prims) : Prop where power2Round : Callee P.power2Round (fun stk => Spec.MlDsa.power2RoundContract X86_64.abi stk) simpleBitPack : Callee P.simpleBitPack (fun stk => Spec.MlDsa.simpleBitPackContract X86_64.abi stk) bitPack : Callee P.bitPack (fun stk => Spec.MlDsa.bitPackContract X86_64.abi stk) + rej4 : Callee4 P.rej4 /-! ## MXCSR -/ @@ -118,7 +126,7 @@ theorem WP.callMx {n : String} {c : Prog isa} {k : Contract isa} /-- `glueCall_ok`, keeping MXCSR's control bits. -/ theorem glueCallMx_ok {glue : List Instr} {n : String} {c : Prog isa} {k : Contract isa} (hv : ∀ s, k.pre s → ∃ t s', Exec isa c s t s' ∧ abiPreserved s s' ∧ k.post s s') - (hsp : NoSp c) (hd : c.depth ≤ 2) (hgl : ∀ i ∈ glue, loadsMxcsr i = false) {s : State} + (hsp : NoSp c) (hd : c.depth ≤ 3) (hgl : ∀ i ∈ glue, loadsMxcsr i = false) {s : State} {V : State → Prop} (hg : WP isa (.block glue) s fun s1 => (V s1 ∧ s1.mem = s.mem) ∧ Keep MlKem.X86_64.argRegs s s1) {rd wr : List Region} (hpre : ∀ s1, V s1 → s1.mem = s.mem → Keep MlKem.X86_64.argRegs s s1 → diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean index f8a03397c..8b7a2eed6 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean @@ -44,6 +44,19 @@ theorem ceWf (h24 : 24 ≤ (s.gpr .rsp).toNat) {n : Nat} (hn : n + 1 ≤ 16) : n have := (s.gpr .rsp).isLt omega +omit L in +theorem ceWf24 (h32 : 32 ≤ (s.gpr .rsp).toNat) : 24 ≤ (s1.gpr .rsp - 8).toNat := by + rw [hsp, BitVec.toNat_sub] + have : (8 : BitVec 64).toNat = 8 := rfl + rw [this] + have := (s.gpr .rsp).isLt + omega + +theorem ceD24 {q : Ptr} {l : Nat} (h : inB (kgB p) q l = true) : + Region.Disjoint (below (s1.gpr .rsp - 8) 24) ⟨pa s q, l⟩ := by + have := stk_disj24 s1 (R := ⟨pa s q, l⟩) (by rw [hsp]; exact L.stkD h) + simpa only [State.callEntry_rsp] using this + end /-! ## The memory of a call's entry -/ @@ -93,7 +106,7 @@ theorem primOk {c : Prog isa} {kk : Nat → Contract isa} (hc : Callee c kk) {gl ∃ s₂ : State, s₂.mem = s'.mem ∧ (∀ r, r ≠ .rsp → s₂.gpr r = s'.gpr r) ∧ (kk stk).post (s1.callEntry.withRegions rd wr) s₂ := by obtain ⟨stk, hs, hver⟩ := hc.verified - exact WP.mono (glueCallMx_ok hver.1 hc.nosp hc.depth hgl hg (hpre stk hs) hcov hw) + exact WP.mono (glueCallMx_ok hver.1 hc.nosp (Nat.le_succ_of_le hc.depth) hgl hg (hpre stk hs) hcov hw) fun s' ⟨h1, h2, h3⟩ => ⟨h1, h2, stk, hs, h3⟩ /-- A block of moves leaks nothing. -/ @@ -188,7 +201,9 @@ theorem covers2 {s : State} {p : Params} (L : Lay kgR (kgW p) s) {a b : Ptr} {la /-- A state of the function, where a call can be made. -/ structure Site (p : Params) (s : State) : Prop where lay : Lay kgR (kgW p) s - h24 : 24 ≤ (s.gpr .rsp).toNat + h32 : 32 ≤ (s.gpr .rsp).toNat + +theorem Site.h24 {p : Params} {s : State} (S : Site p s) : 24 ≤ (s.gpr .rsp).toNat := by have := S.h32; omega /-- The registers that hold the pointers of the layout. -/ abbrev kgRegs : List Reg := [.rbx, .rbp, .r12, .r13] @@ -594,6 +609,86 @@ theorem rejNttAt_tr {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.rejNTTCo end +/-! ## `RejNTTPoly` four times -/ + +section +variable {p : Params} {a w : Ptr} (ha : PtrOk a) (hw : PtrOk w) + (h1 : sepB (kgB p) (sc oSA4) 136 a 4096 = true) (h2 : sepB (kgB p) (sc oSA4) 136 w 8192 = true) + (h3 : sepB (kgB p) a 4096 w 8192 = true) + (w1 : inB (kgW p) a 4096 = true) (w2 : inB (kgW p) w 8192 = true) + +include h1 h2 h3 in +theorem rej4_pre {s s1 : State} (S : Site p s) + (hv : s1.gpr .rdi = pa s (sc oSA4) ∧ s1.gpr .rsi = pa s a ∧ s1.gpr .rdx = pa s w) + (k : Keep MlKem.X86_64.argRegs s s1) : + (Spec.MlDsa.rejNTT4Contract X86_64.abi 24).pre + (s1.callEntry.withRegions [⟨pa s (sc oSA4), 136⟩] [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩]) := by + obtain ⟨i1, i2, _⟩ := sepB_spec h1 + obtain ⟨_, i3, _⟩ := sepB_spec h3 + have L := S.lay + have hsp := keep_rsp k + sig_pre [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] + simp only [hv.1, hv.2.1, hv.2.2] + exact ⟨ceWf24 hsp S.h32, trivial, trivial, L.disj h1, L.disj h2, L.disj h3, ceD1 L hsp i1, ceD1 L hsp i2, + ceD1 L hsp i3, ceD24 L hsp i1, ceD24 L hsp i2, ceD24 L hsp i3, L.nwp i1, L.nwp i2, L.nwp i3⟩ + +include ha hw h1 h2 h3 w1 w2 in +theorem rej4At_ok {c : Prog isa} {sfx : String} (hc : Callee4 c) {s : State} (S : Site p s) : + WP isa (rej4At c sfx a w) s fun s' => Post s s' [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩] ∧ MX s' = MX s ∧ + ((s'.gpr .rax).setWidth 32 = 1 → ∀ k < 4, Spec.MlDsa.Reduced s'.mem (Spec.MlDsa.poly4 (pa s a) k)) ∧ + (((s'.gpr .rax).setWidth 32 = 1 ∧ ∀ k < 4, ∃ b : Spec.MlDsa.Bounds, + Spec.MlDsa.rejNTTPoly b.rejNTT (Spec.MlDsa.seed4 s.mem (pa s (sc oSA4)) k) = + some (Spec.MlDsa.polyAt s'.mem (Spec.MlDsa.poly4 (pa s a) k))) ∨ + ((s'.gpr .rax).setWidth 32 = 0 ∧ ∃ k < 4, + Spec.MlDsa.rejNTTPoly Spec.MlDsa.minBounds.rejNTT (Spec.MlDsa.seed4 s.mem (pa s (sc oSA4)) k) = none)) := by + obtain ⟨i1, _, _⟩ := sepB_spec h1 + have L := S.lay + refine WP.mono (glueCallMx_ok hc.verified.1 hc.nosp hc.depth + (noLd_append (noLd_append (lea_noLd _ _) (lea_noLd _ _)) (lea_noLd _ _)) + (glue3_ok (sc_ok oSA4 (by decide)) ha hw s) (fun s1 hv _ k => rej4_pre h1 h2 h3 S hv k) + (covers_rww L i1 w1 w2) (covers2 L w1 w2)) + fun s' ⟨hP, hx, s1, hv, hm, k, s₂, hm₂, hg₂, hpost⟩ => ⟨hP, hx, ?_⟩ + have hsp := keep_rsp k + sig_post [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] at hpost + simp only [hv.1, hv.2.1, hm₂, hg₂ .rax (by decide)] at hpost + have hseed : ∀ k < 4, Spec.MlDsa.seed4 (ceM s1) (pa s (sc oSA4)) k = Spec.MlDsa.seed4 s.mem (pa s (sc oSA4)) k := + fun k hk => by + unfold Spec.MlDsa.seed4 + rw [← hm] + refine Proof.MlKem.bytesAt_congr fun i hi => ?_ + rw [BitVec.add_assoc, ← BitVec.ofNat_add] + exact callEntry_bytes s1 (R := ⟨pa s (sc oSA4), 136⟩) (k16 s1 (by rw [hsp]; exact L.stkD i1)) + (show 136 ≤ 2 ^ 64 by decide) (show 34 * k + i < 136 by omega) + obtain ⟨hr, ho⟩ := hpost + refine ⟨hr, ?_⟩ + rcases ho with ⟨h1', hb⟩ | ⟨h0, k, hk, hn⟩ + · exact .inl ⟨h1', fun k hk => by rw [← hseed k hk]; exact hb k hk⟩ + · exact .inr ⟨h0, k, hk, by rw [← hseed k hk]; exact hn⟩ + +include ha hw h1 h2 h3 w1 w2 in +theorem rej4At_tr {c : Prog isa} {sfx : String} (hc : Callee4 c) (hba : a.1 ∈ kgRegs) (hbw : w.1 ∈ kgRegs) : + RelCT isa (fun x y => Two p x y ∧ bytesAt x.mem (pa x (sc oSA4)) 136 = bytesAt y.mem (pa y (sc oSA4)) 136) + (rej4At c sfx a w) fun _ _ => True := by + obtain ⟨i1, _, _⟩ := sepB_spec h1 + have g := fun x => glue3_ok (sc_ok oSA4 (by decide)) ha hw x + refine glueCall_tr hc.verified.1 hc.verified.2.1 + (moves_tr (nomem_append (nomem_append (lea_nomem _ _) (lea_nomem _ _)) (lea_nomem _ _))) + (fun x y _ => ⟨g x, g y⟩) + fun x y x1 y1 ⟨T, e⟩ ⟨⟨hv1, hm1⟩, k1⟩ ⟨⟨hv2, hm2⟩, k2⟩ => + ⟨_, _, _, _, rej4_pre h1 h2 h3 T.sx hv1 k1, rej4_pre h1 h2 h3 T.sy hv2 k2, ?_, + by rw [k1.2.1, k1.2.2]; exact covers_rww T.sx.lay i1 w1 w2, by rw [k1.2.2]; exact covers2 T.sx.lay w1 w2, + by rw [k2.2.1, k2.2.2]; exact covers_rww T.sy.lay i1 w1 w2, by rw [k2.2.2]; exact covers2 T.sy.lay w1 w2, + by rw [keep_rsp k1, keep_rsp k2, T.rsp]⟩ + sig_pub [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] + simp only [hv1.1, hv1.2.1, hv1.2.2, hv2.1, hv2.2.1, hv2.2.2, T.pa hba, T.pa hbw, + T.pa (q := sc oSA4) (by decide), and_true] + refine ⟨by rw [keep_rsp k1, keep_rsp k2, T.rsp], ?_⟩ + rw [T.pa (q := sc oSA4) (by decide)] at e + rw [ce_bytesAt' (by decide) (by rw [keep_rsp k1, ← T.pa (q := sc oSA4) (by decide)]; exact T.sx.lay.stkD i1), + ce_bytesAt' (by decide) (by rw [keep_rsp k2]; exact T.sy.lay.stkD i1), hm1, hm2, e] + +end + /-! ## Moves with immediates -/ theorem w32_toNat {v : Nat} (h : v < 2 ^ 32) : (BitVec.setWidth 32 (BitVec.ofNat 64 v)).toNat = v := by diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean index 0713c4074..85c4fa302 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean @@ -16,8 +16,9 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Same # ML-DSA key generation on x86-64, with this library's primitives Untrusted: everything here is checked by Lean. The x86-64 implementations of -the primitives (`prims`) are verified, use at most 16 bytes of stack, and -never write `rsp` but by calls nested at most twice, with any implementation +the primitives (`prims`) are verified, use at most 16 bytes of stack (24 +for `vg_mldsa_rej_ntt_poly4`), and never write `rsp` but by calls nested at +most twice (three times for `vg_mldsa_rej_ntt_poly4`), with any implementation `v` of the polynomial arithmetic (`prims_okWith`), so key generation with them is verified (`keyGen_verifiedWith`). -/ @@ -50,6 +51,7 @@ theorem prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) where by decide +kernel⟩ : Callee prims.simpleBitPack _) bitPack := (⟨⟨0, by decide, Pack.bitPack_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ : Callee prims.bitPack _) + rej4 := ⟨v.ok.rej4.ver, v.ok.rej4.nosp, v.ok.rej4.depth⟩ /-! For any implementation of the polynomial arithmetic, that key generation never writes `rsp` is checked by evaluating it with every @@ -57,7 +59,7 @@ function of it empty (`keyGen_same`, as `sign_same`). -/ theorem keyGen_same {m mc : Prog isa → Bool} (hm : Comp m mc) {B : Backend} (h1 : mc B.ntt = true) (h2 : mc B.invNtt = true) (h3 : mc B.mul = true) (h4 : mc B.mulAdd = true) (h5 : mc B.add = true) - (p : Spec.MlDsa.Params) : Same m (keyGen (primsWith B) p) (keyGen (primsWith .empty) p) := by + (h6 : mc B.rej4 = true) (p : Spec.MlDsa.Params) : Same m (keyGen (primsWith B) p) (keyGen (primsWith .empty) p) := by unfold keyGen same_tac hm @@ -73,7 +75,8 @@ include hp theorem keyGen_spSafe : (keyGen (primsWith v.code) p).all (fun i => !isa.writesSp i) = true := Code.all_of_allInstrs (Same.ok (keyGen_same (Comp.all _) (Code.allInstrs_of_all v.ok.ntt.sp) (Code.allInstrs_of_all v.ok.invNtt.sp) (Code.allInstrs_of_all v.ok.mul.sp) - (Code.allInstrs_of_all v.ok.mulAdd.sp) (Code.allInstrs_of_all v.ok.add.sp) p) (keyGen0_sp hp)) + (Code.allInstrs_of_all v.ok.mulAdd.sp) (Code.allInstrs_of_all v.ok.add.sp) + (Code.allInstrs_of_all v.ok.rej4.sp) p) (keyGen0_sp hp)) theorem keyGen_verifiedWith : Verified X86_64.target (keyGen (primsWith v.code) p) (Spec.MlDsa.keyGenContract p X86_64.abi 32) := diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inv.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inv.lean index 4dd5e19c1..24f2939c2 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inv.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inv.lean @@ -43,7 +43,9 @@ def kcChk (p : Params) (ws : List (Ptr × Nat)) : Bool := topChk (kgB p) ws && k /-- A piece that writes `ws` keeps `K1`. -/ def k1Chk (p : Params) (ws : List (Ptr × Nat)) : Bool := kcChk p ws && keepB (kgB p) ws (sc oHX) 128 && keepB (kgB p) ws (sc oSA) 32 && keepB (kgB p) ws (sc oSB) 64 && - keepB (kgB p) ws (sc (oSB + 65)) 1 + keepB (kgB p) ws (sc (oSB + 65)) 1 && keepB (kgB p) ws (sc (oSA4 + 34 * 0)) 32 && + keepB (kgB p) ws (sc (oSA4 + 34 * 1)) 32 && keepB (kgB p) ws (sc (oSA4 + 34 * 2)) 32 && + keepB (kgB p) ws (sc (oSA4 + 34 * 3)) 32 section variable {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Lay.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Lay.lean index 93212f2a2..9c0a9b49d 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Lay.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Lay.lean @@ -42,7 +42,7 @@ abbrev scrLen (p : Params) : Nat := scratchWords p * 8 /-- `vg_mldsa*_keygen(seed = rdi, pk = rsi, sk = rdx, scratch = rcx) -> eax`, with 32 bytes of stack. -/ def kgK (p : Params) : Contract isa where pre s := - 24 ≤ (s.gpr .rsp).toNat ∧ + 32 ≤ (s.gpr .rsp).toNat ∧ s.rd = [⟨s.gpr .rdi, 32⟩] ∧ s.wr = [⟨s.gpr .rsi, p.pkLen⟩, ⟨s.gpr .rdx, p.skLen⟩, ⟨s.gpr .rcx, scrLen p⟩] ∧ Region.Disjoint ⟨s.gpr .rdi, 32⟩ ⟨s.gpr .rsi, p.pkLen⟩ ∧ Region.Disjoint ⟨s.gpr .rdi, 32⟩ ⟨s.gpr .rdx, p.skLen⟩ ∧ Region.Disjoint ⟨s.gpr .rdi, 32⟩ ⟨s.gpr .rcx, scrLen p⟩ ∧ @@ -188,6 +188,7 @@ macro_rules set_option linter.unusedSimpArgs false in try simp only [VG.Proof.MlDsa.X86_64.KeyGen.scrLen, VG.Spec.MlDsa.scratchWords, VG.Impl.MlDsa.X86_64.KeyGen.oP, VG.Impl.MlDsa.X86_64.KeyGen.oSA, VG.Impl.MlDsa.X86_64.KeyGen.oSB, + VG.Impl.MlDsa.X86_64.KeyGen.oSA4, VG.Impl.MlDsa.X86_64.KeyGen.oR4, VG.Impl.MlDsa.X86_64.KeyGen.oHX, VG.Impl.MlDsa.X86_64.KeyGen.oKL, VG.Impl.MlKem.X86_64.oSS, VG.Impl.MlKem.X86_64.oSV, VG.Impl.MlDsa.X86_64.KeyGen.oT0, $ls,*] and_intros <;> omega_arith)) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Mask.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Mask.lean index 2a071d76a..06af58f26 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Mask.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Mask.lean @@ -19,7 +19,8 @@ open VG.Spec.MlDsa (Params coeffAt) /-! ## Coefficients in memory -/ -theorem coeffAt_writeW32 (m : Mem) (q : Addr) {i j : Nat} (hi : i < 256) (hj : j < 256) (v : BitVec 32) : +theorem coeffAt_writeW32 (m : Mem) (q : Addr) {N i j : Nat} (hN : N ≤ 2 ^ 20) (hi : i < N) (hj : j < N) + (v : BitVec 32) : coeffAt (m.writeW (q + BitVec.ofNat 64 (4 * j)) v) q i = if j = i then v else coeffAt m q i := by unfold coeffAt split @@ -50,38 +51,39 @@ theorem and_mask {r : BitVec 32} (hr : r = 0 ∨ r = 1) (x : BitVec 32) : · rw [ifp rfl, show (BitVec.setWidth 64 (0 - (1 : BitVec 32))).setWidth 32 = BitVec.allOnes 32 by decide] exact BitVec.and_allOnes -abbrev maskPre (a : Ptr) : List Instr := - [.alu32 .and .r15 (.reg .rax), .mov32 .r8 (.imm 0), .alu32 .sub .r8 (.reg .rax)] ++ lea .rdi a ++ imm .rcx 256 +abbrev maskPre (a : Ptr) (N : Nat) : List Instr := + [.alu32 .and .r15 (.reg .rax), .mov32 .r8 (.imm 0), .alu32 .sub .r8 (.reg .rax)] ++ lea .rdi a ++ imm .rcx N -theorem maskPre_ok {a : Ptr} (ha : PtrOk a) (h15 : a.1 ≠ .r15) (s : State) : - WP isa (.block (maskPre a)) s fun s' => +theorem maskPre_ok {a : Ptr} (ha : PtrOk a) (h15 : a.1 ≠ .r15) {N : Nat} (hN : N < 2 ^ 32) (s : State) : + WP isa (.block (maskPre a N)) s fun s' => (s'.mem = s.mem ∧ s'.gpr .r15 = BitVec.setWidth 64 ((s.gpr .r15).setWidth 32 &&& (s.gpr .rax).setWidth 32) ∧ s'.gpr .r8 = BitVec.setWidth 64 (0 - (s.gpr .rax).setWidth 32) ∧ s'.gpr .rdi = pa s a ∧ - s'.gpr .rcx = BitVec.ofNat 64 256) ∧ Keep [.r15, .r8, .rdi, .rcx] s s' := by + s'.gpr .rcx = BitVec.ofNat 64 N) ∧ Keep [.r15, .r8, .rdi, .rcx] s s' := by refine WP.keep _ ?_ (by rfl) unfold maskPre lea imm - xrun [sx_ofNat ha.off, imm_eq (show 256 < 2 ^ 32 by decide), h15, ha.ne (r := .r8) (by decide), + xrun [sx_ofNat ha.off, imm_eq hN, h15, ha.ne (r := .r8) (by decide), ha.ne (r := .rdi) (by decide), List.cons_append, List.nil_append] -theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha : PtrOk a) (ha15 : a.1 ≠ .r15) - (w : inB (kgW p) a 1024 = true) (hr : (s.gpr .rax).setWidth 32 = 0 ∨ (s.gpr .rax).setWidth 32 = 1) : - WP isa (mask a) s fun s' => PostB s s' [⟨pa s a, 1024⟩] ∧ MX s' = MX s ∧ +theorem maskN_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha : PtrOk a) (ha15 : a.1 ≠ .r15) + {N : Nat} (hN0 : 0 < N) (hN : N ≤ 2 ^ 20) (w : inB (kgW p) a (4 * N) = true) + (hr : (s.gpr .rax).setWidth 32 = 0 ∨ (s.gpr .rax).setWidth 32 = 1) : + WP isa (mask a N) s fun s' => PostB s s' [⟨pa s a, 4 * N⟩] ∧ MX s' = MX s ∧ s'.gpr .r15 = BitVec.setWidth 64 ((s.gpr .r15).setWidth 32 &&& (s.gpr .rax).setWidth 32) ∧ - ∀ i < 256, coeffAt s'.mem (pa s a) i = + ∀ i < N, coeffAt s'.mem (pa s a) i = if (s.gpr .rax).setWidth 32 = 1 then coeffAt s.mem (pa s a) i else 0 := by - have hW : InRegions s.wr (pa s a) 1024 := L.cW w _ _ ⟨_, List.mem_singleton_self _, Region.contains_self _ _⟩ - have hn : (pa s a).toNat + 1024 ≤ 2 ^ 64 := L.nwp (inB_mono w) - refine WP.mono (WP.mx (c := mask a) (noLd_spec (by rfl)) (Q := fun s' => PostB s s' [⟨pa s a, 1024⟩] ∧ + have hW : InRegions s.wr (pa s a) (4 * N) := L.cW w _ _ ⟨_, List.mem_singleton_self _, Region.contains_self _ _⟩ + have hn : (pa s a).toNat + 4 * N ≤ 2 ^ 64 := L.nwp (inB_mono w) + refine WP.mono (WP.mx (c := mask a N) (noLd_spec (by rfl)) (Q := fun s' => PostB s s' [⟨pa s a, 4 * N⟩] ∧ s'.gpr .r15 = BitVec.setWidth 64 ((s.gpr .r15).setWidth 32 &&& (s.gpr .rax).setWidth 32) ∧ - ∀ i < 256, coeffAt s'.mem (pa s a) i = + ∀ i < N, coeffAt s'.mem (pa s a) i = if (s.gpr .rax).setWidth 32 = 1 then coeffAt s.mem (pa s a) i else 0) ?_) fun s' ⟨⟨hP, h15, hc⟩, hx⟩ => ⟨hP, hx, h15, hc⟩ unfold mask - refine WP.seq (WP.mono (maskPre_ok ha ha15 s) fun s1 ⟨⟨hm1, h15, h8, hdi, hcx⟩, k1⟩ => ?_) - refine WP.mono (wp_countdown (cnt := .rcx) (N := 256) (by decide) (by decide) (fun i s' => + refine WP.seq (WP.mono (maskPre_ok ha ha15 (N := N) (by omega) s) fun s1 ⟨⟨hm1, h15, h8, hdi, hcx⟩, k1⟩ => ?_) + refine WP.mono (wp_countdown (cnt := .rcx) (N := N) (by omega) hN0 (fun i s' => s'.gpr .rdi = pa s a + BitVec.ofNat 64 (4 * i) ∧ s'.gpr .r8 = s1.gpr .r8 ∧ s'.gpr .r15 = s1.gpr .r15 ∧ - s'.rd = s.rd ∧ s'.wr = s.wr ∧ Frame [⟨pa s a, 1024⟩] s.mem s'.mem ∧ - (∀ j < 256, coeffAt s'.mem (pa s a) j = + s'.rd = s.rd ∧ s'.wr = s.wr ∧ Frame [⟨pa s a, 4 * N⟩] s.mem s'.mem ∧ + (∀ j < N, coeffAt s'.mem (pa s a) j = if j < i then coeffAt s.mem (pa s a) j &&& (s1.gpr .r8).setWidth 32 else coeffAt s.mem (pa s a) j) ∧ Keep [.r15, .r8, .rdi, .rcx, .rax] s s') (fun i hi s' ⟨hdi', h8', h15', hrd', hwr', hf, hc, kk⟩ _ => ?_) (fun _ h => h) @@ -89,7 +91,7 @@ theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha : fun j _ => by rw [ifn (Nat.not_lt_zero j), hm1], k1.mono (by decide)⟩ hcx) fun s' ⟨_, h8', h15', hrd', hwr', hf, hc, kk⟩ => ⟨⟨hrd', hwr', fun r hr => kk.gpr ?_, kk.gpr (by decide), hf.mono fun _ hr => List.mem_append_left _ hr⟩, ?_, ?_⟩ - · have hc4 : (⟨pa s a, 1024⟩ : Region).Contains (pa s a + BitVec.ofNat 64 (4 * i)) 4 := + · have hc4 : (⟨pa s a, 4 * N⟩ : Region).Contains (pa s a + BitVec.ofNat 64 (4 * i)) 4 := contains_offset' (by omega) (by omega) have hin : InRegions s'.wr (s'.gpr .rdi) 4 := by rw [hwr', hdi']; exact inRegions_sub hW (by omega) (by omega) @@ -101,7 +103,7 @@ theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha : (kk.trans k').mono (by decide)⟩, hcx'', hz⟩ · rw [hdi'', hdi', show (4 : BitVec 64) = BitVec.ofNat 64 4 from rfl, off_add]; rfl · rw [hm, hdi']; exact hf.writeW (List.mem_singleton_self _) _ hc4 - · rw [hm, hdi', coeffAt_writeW32 _ _ hj (by omega), h8'] + · rw [hm, hdi', coeffAt_writeW32 _ _ hN hj (by omega), h8'] by_cases e : i = j · subst e rw [ifp rfl, ifp (Nat.lt_succ_self _)] @@ -117,6 +119,14 @@ theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha : · rw [h15', h15] · intro j hj rw [hc j hj, ifp hj, h8, and_mask hr] +theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha : PtrOk a) (ha15 : a.1 ≠ .r15) + (w : inB (kgW p) a 1024 = true) (hr : (s.gpr .rax).setWidth 32 = 0 ∨ (s.gpr .rax).setWidth 32 = 1) : + WP isa (mask a) s fun s' => PostB s s' [⟨pa s a, 1024⟩] ∧ MX s' = MX s ∧ + s'.gpr .r15 = BitVec.setWidth 64 ((s.gpr .r15).setWidth 32 &&& (s.gpr .rax).setWidth 32) ∧ + ∀ i < 256, coeffAt s'.mem (pa s a) i = + if (s.gpr .rax).setWidth 32 = 1 then coeffAt s.mem (pa s a) i else 0 := + maskN_ok L ha ha15 (N := 256) (by decide) (by decide) w hr + /-! ## Constant time -/ /-- The check is the same for every offset: its hint is computed once, for offset 0. -/ @@ -128,4 +138,13 @@ theorem mask_tr {j : Nat} (hj : j < 128) {P : State → State → Prop} (h : ∀ taintRel [.rbx] (fun x y hp r hr => by simp only [List.mem_singleton] at hr; subst hr; exact h x y hp) (mask_taint j hj) +/-- The same for the four polynomials from `oP j`. -/ +theorem mask4_taint : ∀ j < 128, (taint.check (X86_64.Taint.ofRegs [.rbx]) (mask (sc (oP j)) 1024) + (VG.Taint.hintOf taint (X86_64.Taint.ofRegs [.rbx]) (mask (sc 0) 1024))).isSome = true := by decide +kernel + +theorem mask4_tr {j : Nat} (hj : j < 128) {P : State → State → Prop} (h : ∀ x y, P x y → x.gpr .rbx = y.gpr .rbx) : + RelCT isa P (mask (sc (oP j)) 1024) fun _ _ => True := + taintRel [.rbx] (fun x y hp r hr => by simp only [List.mem_singleton] at hr; subst hr; exact h x y hp) + (mask4_taint j hj) + end VG.Proof.MlDsa.X86_64.KeyGen diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestBase.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestBase.lean index d8133e70a..ecc0c8bd2 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestBase.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestBase.lean @@ -1,4 +1,4 @@ -import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Samp +import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Samp4 import VerifiedGarbage.Proof.MlDsa.KeyGen.Rest /-! diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp.lean index 7fa5dc6c2..e66a27665 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp.lean @@ -202,8 +202,8 @@ theorem rej_pub {p : Params} {σ₁ σ₂ : State} (pub : (kgK p).pub σ₁ σ theorem Two.post {p : Params} {x y x' y' : State} (T : Two p x y) {W₁ W₂ : List Region} (hx : PostB x x' W₁) (hy : PostB y y' W₂) : Two p x' y' := - ⟨⟨T.sx.lay.post hx (kgB_bases p), by rw [hx.rsp]; exact T.sx.h24⟩, - ⟨T.sy.lay.post hy (kgB_bases p), by rw [hy.rsp]; exact T.sy.h24⟩, + ⟨⟨T.sx.lay.post hx (kgB_bases p), by rw [hx.rsp]; exact T.sx.h32⟩, + ⟨T.sy.lay.post hy (kgB_bases p), by rw [hy.rsp]; exact T.sy.h32⟩, fun r hr => by have hb : r ∈ bases := by simp only [kgRegs, List.mem_cons, List.not_mem_nil, or_false] at hr rcases hr with rfl | rfl | rfl | rfl <;> decide @@ -400,14 +400,6 @@ theorem KSamp.zero {p : Params} {σ s : State} (h : K1 p σ s) (h15 : s.gpr .r15 fun _ h => absurd h (Nat.not_lt_zero _), .inl ⟨h15, ⟨0, 0, 0, 0⟩, fun _ h => absurd h (Nat.not_lt_zero _), fun _ h => absurd h (Nat.not_lt_zero _)⟩⟩ -/-- The entries of `Â`. -/ -theorem sampA_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) : - Piece p (fun σ s => K1 p σ s ∧ s.gpr .r15 = 1) (fun σ s => KSamp p σ (p.k * p.ℓ) 0 s) - (seqR (expA P p) 0 (p.k * p.ℓ)) := by - refine Piece.mono (Piece.seqR (I := fun e σ s => KSamp p σ e 0 s) (p.k * p.ℓ) 0 - fun e _ he => expA_piece hP hF (by omega)) (fun σ s _ h => KSamp.zero h.1 h.2) fun σ s _ h => ?_ - simpa using h - /-- The entries of `s₁ ‖ s₂`. -/ theorem sampS_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) : Piece p (fun σ s => KSamp p σ (p.k * p.ℓ) 0 s) (fun σ s => KSamp p σ (p.k * p.ℓ) (p.ℓ + p.k) s) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp4.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp4.lean new file mode 100644 index 000000000..aab0228cc --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp4.lean @@ -0,0 +1,208 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Samp + +/-! +# ML-DSA key generation on x86-64: four entries of `Â` at a time + +Untrusted: everything here is checked by Lean. `expA4 g` sets the indices of +entries `4g, …, 4g + 3` of `Â` in the four seeds at `oSA4` (`slot_piece`), +samples the four polynomials with one call of `vg_mldsa_rej_ntt_poly4`, and +masks them with its result (`call_piece`): it takes `KSamp (4g)` to +`KSamp (4g + 4)` (`expA4_piece`), and `expAll`, the groups and then the +last `kℓ mod 4` entries one at a time, takes `K1` to `KSamp (kℓ)` +(`sampAll_piece`). +-/ + +namespace VG.Proof.MlDsa.X86_64.KeyGen + +open VG VG.X86_64 VG.Proof.MlKem.X86_64 +open VG.Impl.MlKem.X86_64 (Ptr sc setB seqR) +open VG.Impl.MlDsa.X86_64.KeyGen +open VG.Spec.MlDsa (Params Poly rejNTTPoly coeffAt polyAt Reduced PolyIs poly4 seed4 minBounds) +open VG.Proof.MlDsa.KeyGen (seedA) +open VG.Spec.Sha3 (bytesAt) + +/-! ## The seeds -/ + +/-- After the first `e` entries of `Â`, with the seeds of entries `e, …, e + j - 1` at `oSA4`. -/ +structure GS (p : Params) (σ : State) (e j : Nat) (s : State) : Prop where + ks : KSamp p σ e 0 s + done : ∀ k < j, bytesAt s.mem (pa s (sc (oSA4 + 34 * k))) 34 = seedA (rhoOf p σ) ((e + k) / p.ℓ) ((e + k) % p.ℓ) + +theorem slot_ok {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) {e j : Nat} (he : e + 4 ≤ p.k * p.ℓ) + (hj : j < 4) {s : State} (h : GS p σ e j s) : WP isa (.block (setSR p e j)) s (GS p σ e (j + 1)) := by + have hkl := hF.kl; have hl := hF.l; have hk := hF.k + have L := h.ks.k1.kc.lay hF hp + have hq : (e + j) / p.ℓ < 256 := Nat.lt_of_le_of_lt (Nat.div_le_self _ _) (by omega) + have hr : (e + j) % p.ℓ < 256 := Nat.lt_of_lt_of_le (Nat.mod_lt _ (by omega)) (by omega) + unfold setSR + refine WP.mono (setTwo_ok L (o := oSA4 + 34 * j + 32) hr hq (by lay) (by lay) (by lay)) + fun s' ⟨hP, hx, hb⟩ => ⟨h.ks.keep hF hp hP.b hx (by layk) (fun e' he' => by layk) + (fun _ h => absurd h (Nat.not_lt_zero _)) (hP.cs .r15 (by decide)), fun k hk => ?_⟩ + rcases (by omega : k < j ∨ k = j) with hk' | rfl + · rw [L.keepBytes hP.b (by layk)]; exact h.done k hk' + · rw [show 34 = 32 + 2 from rfl, Proof.MlKem.bytesAt_add, L.keepBytes hP.b (by layk), h.ks.k1.sa4 k hj, + show pa s' (sc (oSA4 + 34 * k)) + BitVec.ofNat 64 32 = pa s' (sc (oSA4 + 34 * k + 32)) from off_add _ _ _, + hP.pa rbx_cs, hb, seedA_eq] + +theorem slot_piece {p : Params} (hF : PFacts p) {e j : Nat} (he : e + 4 ≤ p.k * p.ℓ) (hj : j < 4) : + Piece p (GS p · e j) (GS p · e (j + 1)) (.block (setSR p e j)) := + ⟨fun _ _ hp h => slot_ok hF hp he hj h, + rel_of (Q := Two p) (taintRel [.rbx] (fun x y h => two_rbx h) (hc := .block []) (by with_unfolding_all rfl)) + fun _ _ _ _ p₁ p₂ pub h₁ h₂ => kc_two hF p₁ p₂ pub h₁.ks.k1.kc h₂.ks.k1.kc⟩ + +theorem seed4_eq {p : Params} {σ : State} {e : Nat} {s : State} (h : GS p σ e 4 s) {k : Nat} (hk : k < 4) : + seed4 s.mem (pa s (sc oSA4)) k = seedA (rhoOf p σ) ((e + k) / p.ℓ) ((e + k) % p.ℓ) := by + unfold seed4 + rw [show pa s (sc oSA4) + BitVec.ofNat 64 (34 * k) = pa s (sc (oSA4 + 34 * k)) from off_add _ _ _] + exact h.done k hk + +theorem bytes136 (m : Mem) (P : Addr) : bytesAt m P 136 = bytesAt m P 34 ++ bytesAt m (P + BitVec.ofNat 64 34) 34 ++ + bytesAt m (P + BitVec.ofNat 64 68) 34 ++ bytesAt m (P + BitVec.ofNat 64 102) 34 := by + rw [show 136 = 34 + 102 from rfl, Proof.MlKem.bytesAt_add, show 102 = 34 + 68 from rfl, Proof.MlKem.bytesAt_add, + show 68 = 34 + 34 from rfl, Proof.MlKem.bytesAt_add] + simp only [BitVec.add_assoc, ← BitVec.ofNat_add, List.append_assoc, Nat.reduceAdd] + +/-- The four seeds are those of the entries, from `ρ`. -/ +theorem GS.seeds {p : Params} {σ : State} {e : Nat} {s : State} (h : GS p σ e 4 s) : + bytesAt s.mem (pa s (sc oSA4)) 136 = seedA (rhoOf p σ) ((e + 0) / p.ℓ) ((e + 0) % p.ℓ) ++ + seedA (rhoOf p σ) ((e + 1) / p.ℓ) ((e + 1) % p.ℓ) ++ seedA (rhoOf p σ) ((e + 2) / p.ℓ) ((e + 2) % p.ℓ) ++ + seedA (rhoOf p σ) ((e + 3) / p.ℓ) ((e + 3) % p.ℓ) := by + have b : ∀ k < 4, bytesAt s.mem (pa s (sc oSA4) + BitVec.ofNat 64 (34 * k)) 34 = + seedA (rhoOf p σ) ((e + k) / p.ℓ) ((e + k) % p.ℓ) := fun k hk => seed4_eq h hk + have b0 := b 0 (by decide) + rw [show 34 * 0 = 0 from rfl, add_ofNat_zero] at b0 + rw [bytes136, b0, b 1 (by decide), b 2 (by decide), b 3 (by decide)] + +/-! ## The four polynomials -/ + +theorem coeffAt_poly4 (m : Mem) (a : Addr) (k j : Nat) : coeffAt m (poly4 a k) j = coeffAt m a (256 * k + j) := by + unfold coeffAt poly4 + rw [BitVec.add_assoc, ← BitVec.ofNat_add, show 1024 * k + 4 * j = 4 * (256 * k + j) by omega] + +theorem pa_poly4 (s : State) (e k : Nat) : poly4 (pa s (aP e)) k = pa s (aP (e + k)) := by + unfold poly4 + rw [show pa s (aP e) + BitVec.ofNat 64 (1024 * k) = pa s (sc (oP e + 1024 * k)) from off_add _ _ _, + show oP e + 1024 * k = oP (e + k) by simp only [oP]; omega] + +/-- One bound for the four polynomials. -/ +theorem bound4 {ρ : Nat → List Byte} {x : Nat → Poly} (h : ∀ k < 4, ∃ b : Spec.MlDsa.Bounds, + rejNTTPoly b.rejNTT (ρ k) = some (x k)) : ∃ n : Nat, ∀ k < 4, rejNTTPoly n (ρ k) = some (x k) := by + obtain ⟨b0, h0⟩ := h 0 (by decide) + obtain ⟨b1, h1⟩ := h 1 (by decide) + obtain ⟨b2, h2⟩ := h 2 (by decide) + obtain ⟨b3, h3⟩ := h 3 (by decide) + refine ⟨max (max b0.rejNTT b1.rejNTT) (max b2.rejNTT b3.rejNTT), fun k hk => ?_⟩ + rcases (by omega : k = 0 ∨ k = 1 ∨ k = 2 ∨ k = 3) with rfl | rfl | rfl | rfl + · exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (by omega) h0 + · exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (by omega) h1 + · exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (by omega) h2 + · exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (by omega) h3 + +theorem call_ok {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) + {g : Nat} (hg : 4 * g + 4 ≤ p.k * p.ℓ) {s : State} (h : GS p σ (4 * g) 4 s) : + WP isa (.seq (rej4At P.rej4 P.sfx (aP (4 * g)) (sc (oR4 p))) (mask (aP (4 * g)) 1024)) s + (KSamp p σ (4 * g + 4) 0) := by + have hkl := hF.kl; have hl := hF.l; have hk := hF.k + have S := h.ks.k1.kc.site hF hp + have L := S.lay + refine WP.seq (WP.mono (rej4At_ok (a := aP (4 * g)) (w := sc (oR4 p)) (sc_ok _ (by simp only [oP]; omega)) + (sc_ok _ (by simp only [oR4, oP]; omega)) (by lay) (by lay) (by lay) (by lay) (by lay) hP.rej4 S) + fun s₂ ⟨hP₂, hx₂, hred, hout⟩ => ?_) + have hP₂' : PPostB s s₂ [(aP (4 * g), 4096), (sc (oR4 p), 8192)] := hP₂.b + have L₂ := L.post hP₂' (kgB_bases p) + have hr01 : (s₂.gpr .rax).setWidth 32 = 0 ∨ (s₂.gpr .rax).setWidth 32 = 1 := by + rcases hout with ⟨h1, _⟩ | ⟨h0, _⟩ + exacts [.inr h1, .inl h0] + refine WP.mono (maskN_ok L₂ (a := aP (4 * g)) (N := 1024) (sc_ok _ (by simp only [oP]; omega)) + (show Reg.rbx ≠ .r15 by decide) (by decide) (by decide) (by lay) hr01) fun s₃ ⟨hP₃, hx₃, h15, hco⟩ => ?_ + have hP₃' : PPostB s₂ s₃ [(aP (4 * g), 4 * 1024)] := hP₃ + have hP₁₃ := PPostB.app hP₂' hP₃' (sc1_bases _ _) + have e₂ : pa s₂ (aP (4 * g)) = pa s (aP (4 * g)) := hP₂'.pa rbx_bases + rw [e₂] at hco + have hco4 : ∀ k < 4, ∀ i < 256, coeffAt s₃.mem (poly4 (pa s (aP (4 * g))) k) i = + if (s₂.gpr .rax).setWidth 32 = 1 then coeffAt s₂.mem (poly4 (pa s (aP (4 * g))) k) i else 0 := + fun k hk i hi => by rw [coeffAt_poly4, coeffAt_poly4]; exact hco _ (by omega) + -- Entry `4g + k` is polynomial `k` from `aP (4g)`. + have e₃ : ∀ k, pa s₃ (aP (4 * g + k)) = poly4 (pa s (aP (4 * g))) k := fun k => by + rw [pa_poly4]; exact hP₁₃.pa rbx_bases + obtain ⟨A, S', hA, _, hG⟩ := h.ks.ex + have h15₂ : s₂.gpr .r15 = s.gpr .r15 := hP₂.cs .r15 (by decide) + rw [h15₂, r15_and (good_01 hG) hr01] at h15 + refine ⟨h.ks.k1.step hF hp hP₁₃ (hx₃.trans hx₂) (by layk), fun e' => if e' < 4 * g then A e' + else polyAt s₃.mem (pa s₃ (aP e')), S', fun e' he' => ?_, fun _ h => absurd h (Nat.not_lt_zero _), ?_⟩ + · dsimp only + by_cases hlt : e' < 4 * g + · rw [ifp hlt] + exact polyIs_frame' L hP₁₃ (by layk) (hA e' hlt) + · rw [ifn hlt] + refine ⟨?_, rfl⟩ + obtain ⟨k, rfl⟩ : ∃ k, e' = 4 * g + k := ⟨e' - 4 * g, by omega⟩ + rw [e₃] + by_cases h1 : (s₂.gpr .rax).setWidth 32 = 1 + · exact (masked_one h1 (hco4 k (by omega))).2 (hred h1 k (by omega)) + · exact (masked_zero h1 (hco4 k (by omega))).1 + · rw [h15] + rcases hG with ⟨h1, b, hb, _⟩ | ⟨h0, hn⟩ + · rcases hout with ⟨ho, hb4⟩ | ⟨ho, k, hk, hn⟩ + · obtain ⟨n, hn⟩ := bound4 hb4 + refine .inl ⟨by rw [ifp ⟨h1, ho⟩], { b with rejNTT := max b.rejNTT n }, fun e' he' => ?_, + fun _ h => absurd h (Nat.not_lt_zero _)⟩ + dsimp only + by_cases hlt : e' < 4 * g + · rw [ifp hlt] + exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (Nat.le_max_left _ _) (hb e' hlt) + · rw [ifn hlt] + obtain ⟨k, rfl⟩ : ∃ k, e' = 4 * g + k := ⟨e' - 4 * g, by omega⟩ + have hk : k < 4 := by omega + rw [e₃, (masked_one ho (hco4 k hk)).1, ← seed4_eq h hk] + exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (Nat.le_max_right _ _) (hn k hk) + · rw [ifn (fun h => by rw [h.2] at ho; exact absurd ho (by decide))] + rw [seed4_eq h hk] at hn + have hq : (4 * g + k) / p.ℓ < p.k := Nat.div_lt_of_lt_mul (by rw [Nat.mul_comm p.ℓ p.k]; omega) + exact .inr ⟨rfl, Proof.MlDsa.KeyGen.keyGenInternal_none_A hq (Nat.mod_lt _ (by omega)) hn⟩ + · rw [ifn (fun h => by rw [h.1] at h0; exact absurd h0 (by decide))] + exact .inr ⟨rfl, hn⟩ + +theorem call_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) {g : Nat} + (hg : 4 * g + 4 ≤ p.k * p.ℓ) : + Piece p (GS p · (4 * g) 4) (KSamp p · (4 * g + 4) 0) + (.seq (rej4At P.rej4 P.sfx (aP (4 * g)) (sc (oR4 p))) (mask (aP (4 * g)) 1024)) := by + have hkl := hF.kl; have hl := hF.l; have hk := hF.k + refine ⟨fun _ _ hp h => call_ok hP hF hp hg h, rel_of (Q := fun x y => Two p x y ∧ + bytesAt x.mem (pa x (sc oSA4)) 136 = bytesAt y.mem (pa y (sc oSA4)) 136) ?_ + fun _ _ _ _ p₁ p₂ pub h₁ h₂ => ⟨kc_two hF p₁ p₂ pub h₁.ks.k1.kc h₂.ks.k1.kc, + by rw [h₁.seeds, h₂.seeds, rho_pub pub]⟩⟩ + have ok := fun x (S : Site p x) => WP.mono (rej4At_ok (a := aP (4 * g)) (w := sc (oR4 p)) (sfx := P.sfx) + (sc_ok _ (by simp only [oP]; omega)) (sc_ok _ (by simp only [oR4, oP]; omega)) (by lay) (by lay) (by lay) + (by lay) (by lay) hP.rej4 S) fun _ h => (⟨_, h.1.b⟩ : ∃ W, PostB x _ W) + exact RelCT.seq (RelCT.two (fun _ _ h => h.1) (rej4At_tr (sfx := P.sfx) (sc_ok _ (by simp only [oP]; omega)) + (sc_ok _ (by simp only [oR4, oP]; omega)) (by lay) (by lay) (by lay) (by lay) (by lay) hP.rej4 + (show Reg.rbx ∈ kgRegs by decide) (show Reg.rbx ∈ kgRegs by decide)) + fun x y h => ⟨ok x h.1.sx, ok y h.1.sy⟩) + (mask4_tr (j := 4 * g) (by omega) fun x y h => h.regs .rbx (by decide)) + +/-! ## The pieces -/ + +theorem expA4_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) {g : Nat} + (hg : 4 * g + 4 ≤ p.k * p.ℓ) : + Piece p (KSamp p · (4 * g) 0) (KSamp p · (4 * g + 4) 0) (expA4 P p g) := by + unfold expA4 + exact Piece.mono ((slot_piece hF hg (j := 0) (by decide)).seq ((slot_piece hF hg (j := 1) (by decide)).seq + ((slot_piece hF hg (j := 2) (by decide)).seq ((slot_piece hF hg (j := 3) (by decide)).seq (call_piece hP hF hg))))) + (fun _ _ _ h => ⟨h, fun _ h => absurd h (Nat.not_lt_zero _)⟩) fun _ _ _ h => h + +/-- The entries of `Â`. -/ +theorem sampAll_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) : + Piece p (fun σ s => K1 p σ s ∧ s.gpr .r15 = 1) (fun σ s => KSamp p σ (p.k * p.ℓ) 0 s) (expAll P p) := by + unfold expAll + refine Piece.seq (J := fun σ s => KSamp p σ (4 * (p.k * p.ℓ / 4)) 0 s) ?_ ?_ + · refine Piece.mono (Piece.seqR (I := fun g σ s => KSamp p σ (4 * g) 0 s) (p.k * p.ℓ / 4) 0 + fun g _ hg => Piece.mono (expA4_piece hP hF (g := g) (by omega)) (fun _ _ _ h => h) fun _ _ _ h => ?_) + (fun σ s _ h => KSamp.zero h.1 h.2) fun σ s _ h => ?_ + · rw [show 4 * (g + 1) = 4 * g + 4 by omega]; exact h + · simpa using h + · refine Piece.mono (Piece.seqR (I := fun e σ s => KSamp p σ e 0 s) (p.k * p.ℓ % 4) (4 * (p.k * p.ℓ / 4)) + fun e h1 h2 => expA_piece hP hF (by omega)) (fun _ _ _ h => h) fun σ s _ h => ?_ + rwa [show 4 * (p.k * p.ℓ / 4) + p.k * p.ℓ % 4 = p.k * p.ℓ by omega] at h + +end VG.Proof.MlDsa.X86_64.KeyGen diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Seeds.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Seeds.lean index cb6aac8a6..b71079899 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Seeds.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Seeds.lean @@ -28,8 +28,8 @@ theorem Two.lrel {p : Params} {x y : State} (h : Two p x y) : LRel kgR (kgW p) x theorem Two.step {p : Params} {c : Prog isa} (htr : RelCT isa (Two p) c fun _ _ => True) (hok : ∀ x, Site p x → WP isa c x fun x' => ∃ W, PostB x x' W) : RelCT isa (Two p) c (Two p) := RelCT.postDep htr (F := fun x x' => ∃ W, PostB x x' W) (fun x y h => ⟨hok x h.sx, hok y h.sy⟩) - fun x y x' y' h ⟨_, hx⟩ ⟨_, hy⟩ => ⟨⟨h.sx.lay.post hx (kgB_bases p), by rw [hx.rsp]; exact h.sx.h24⟩, - ⟨h.sy.lay.post hy (kgB_bases p), by rw [hy.rsp]; exact h.sy.h24⟩, + fun x y x' y' h ⟨_, hx⟩ ⟨_, hy⟩ => ⟨⟨h.sx.lay.post hx (kgB_bases p), by rw [hx.rsp]; exact h.sx.h32⟩, + ⟨h.sy.lay.post hy (kgB_bases p), by rw [hy.rsp]; exact h.sy.h32⟩, fun r hr => by have hb : r ∈ bases := by simp only [kgRegs, List.mem_cons, List.not_mem_nil, or_false] at hr rcases hr with rfl | rfl | rfl | rfl <;> decide @@ -137,21 +137,30 @@ theorem pro_piece {p : Params} (hF : PFacts p) : /-- `H(ξ ‖ k ‖ ℓ, 128)`. -/ abbrev hxOf (p : Params) (σ : State) : List Byte := Spec.MlDsa.H (xiOf σ ++ integerToBytes p.k 1 ++ integerToBytes p.ℓ 1) 128 -/-- After the seeds. -/ -structure K1 (p : Params) (σ s : State) : Prop where +/-- After the seeds, with `ρ` copied to the first `j` seeds of `oSA4`. -/ +structure K1R (p : Params) (j : Nat) (σ s : State) : Prop where kc : KC p σ s hx : bytesAt s.mem (pa s (sc oHX)) 128 = hxOf p σ sa : bytesAt s.mem (pa s (sc oSA)) 32 = rhoOf p σ sb : bytesAt s.mem (pa s (sc oSB)) 64 = rho'Of p σ z : bytesAt s.mem (pa s (sc (oSB + 65))) 1 = [0] + sa4 : ∀ k < j, bytesAt s.mem (pa s (sc (oSA4 + 34 * k))) 32 = rhoOf p σ + +/-- After the seeds. -/ +abbrev K1 (p : Params) (σ s : State) : Prop := K1R p 4 σ s theorem K1.step {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) {s s' : State} (h : K1 p σ s) {ws : List (Ptr × Nat)} (hP : PPostB s s' ws) (hx : MX s' = MX s) (hc : k1Chk p ws = true) : K1 p σ s' := by simp only [k1Chk, Bool.and_eq_true] at hc - obtain ⟨⟨⟨⟨h0, h1⟩, h2⟩, h3⟩, h4⟩ := hc + obtain ⟨⟨⟨⟨⟨⟨⟨⟨h0, h1⟩, h2⟩, h3⟩, h4⟩, a0⟩, a1⟩, a2⟩, a3⟩ := hc have L := h.kc.lay hF hp - exact ⟨h.kc.step hF hp hP hx h0, by rw [L.keepBytes hP h1]; exact h.hx, by rw [L.keepBytes hP h2]; exact h.sa, - by rw [L.keepBytes hP h3]; exact h.sb, by rw [L.keepBytes hP h4]; exact h.z⟩ + refine ⟨h.kc.step hF hp hP hx h0, by rw [L.keepBytes hP h1]; exact h.hx, by rw [L.keepBytes hP h2]; exact h.sa, + by rw [L.keepBytes hP h3]; exact h.sb, by rw [L.keepBytes hP h4]; exact h.z, fun k hk => ?_⟩ + rcases (by omega : k = 0 ∨ k = 1 ∨ k = 2 ∨ k = 3) with rfl | rfl | rfl | rfl + · rw [L.keepBytes hP a0]; exact h.sa4 0 hk + · rw [L.keepBytes hP a1]; exact h.sa4 1 hk + · rw [L.keepBytes hP a2]; exact h.sa4 2 hk + · rw [L.keepBytes hP a3]; exact h.sa4 3 hk theorem shake31' : BitVec.ofNat 8 31 = Spec.Sha3.shakeSuffix := by decide @@ -192,6 +201,20 @@ theorem setKL_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a b : Nat} (h bytesAt s'.mem (pa s (sc oKL)) 2 = [BitVec.ofNat 8 a, BitVec.ofNat 8 b] := setTwo_ok L ha hb (by lay) (by lay) (by lay) +/-- `ρ` to seed `j` of `oSA4`. -/ +theorem copyR_ok {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) {j : Nat} (hj : j < 4) {s : State} + (h : K1R p j σ s) : + WP isa (copy (sc (oSA4 + 34 * j)) (sc oHX) 32) s fun s' => K1R p (j + 1) σ s' ∧ s'.gpr .r15 = s.gpr .r15 := by + have hk := hF.k; have hl := hF.l + have L := h.kc.lay hF hp + refine WP.mono (copy_okM L (dst := sc (oSA4 + 34 * j)) (src := sc oHX) (n := 32) (by decide) (by layk)) + fun s' ⟨⟨hP, hb⟩, hx⟩ => ⟨⟨h.kc.step hF hp hP.b hx (by layk), by rw [L.keepBytes hP.b (by layk)]; exact h.hx, + by rw [L.keepBytes hP.b (by layk)]; exact h.sa, by rw [L.keepBytes hP.b (by layk)]; exact h.sb, + by rw [L.keepBytes hP.b (by layk)]; exact h.z, fun k hk' => ?_⟩, hP.cs .r15 (by decide)⟩ + rcases (by omega : k < j ∨ k = j) with hk' | rfl + · rw [L.keepBytes hP.b (by layk)]; exact h.sa4 k hk' + · rw [hP.pa rbx_cs, hb, ← Proof.MlKem.bytesAt_take s.mem (pa s (sc oHX)) (show 32 ≤ 128 by decide), h.hx, ← rho_eq] + theorem seeds_ok {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) {s : State} (h : KC p σ s) (h15 : s.gpr .r15 = 1) : WP isa (seeds p) s fun s' => K1 p σ s' ∧ s'.gpr .r15 = 1 := by have hk := hF.k; have hl := hF.l @@ -224,14 +247,21 @@ theorem seeds_ok {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) have hsb : bytesAt s₃.mem (pa s₃ (sc (oHX + 32))) 64 = rho'Of p σ := by rw [rho'_eq, ← hx3, Proof.MlKem.bytesAt_slice _ _ (show 32 + 64 ≤ 128 by decide), pa, pa, off_add] rw [hsb, ← e₄] at hb₄ - refine WP.mono (WP.mx (noLd_spec (by rfl)) (setB_okL L₄ (p := sc (oSB + 65)) (v := 0) (by decide) (by decide) - (by lay))) fun s₅ ⟨⟨hP₅, hb₅⟩, hx₅⟩ => ⟨⟨h₄.step hF hp hP₅.b hx₅ (by layk), ?_, ?_, ?_, ?_⟩, ?_⟩ - · rw [L₄.keepBytes hP₅.b (by layk), L₃.keepBytes hP₄.b (by layk)]; exact hx3 - · rw [L₄.keepBytes hP₅.b (by layk), L₃.keepBytes hP₄.b (by layk)]; exact hb₃ - · rw [L₄.keepBytes hP₅.b (by layk)]; exact hb₄ - · rw [hP₅.pa rbx_cs]; exact hb₅ - · rw [hP₅.cs .r15 (by decide), hP₄.cs .r15 (by decide), hP₃.cs .r15 (by decide), hP₂.cs .r15 (by decide), + refine WP.seq (WP.mono (WP.mx (noLd_spec (by rfl)) (setB_okL L₄ (p := sc (oSB + 65)) (v := 0) (by decide) + (by decide) (by lay))) fun s₅ ⟨⟨hP₅, hb₅⟩, hx₅⟩ => ?_) + have h₅ : K1R p 0 σ s₅ := ⟨h₄.step hF hp hP₅.b hx₅ (by layk), + by rw [L₄.keepBytes hP₅.b (by layk), L₃.keepBytes hP₄.b (by layk)]; exact hx3, + by rw [L₄.keepBytes hP₅.b (by layk), L₃.keepBytes hP₄.b (by layk)]; exact hb₃, + by rw [L₄.keepBytes hP₅.b (by layk)]; exact hb₄, by rw [hP₅.pa rbx_cs]; exact hb₅, + fun _ h => absurd h (Nat.not_lt_zero _)⟩ + have f₅ : s₅.gpr .r15 = 1 := by + rw [hP₅.cs .r15 (by decide), hP₄.cs .r15 (by decide), hP₃.cs .r15 (by decide), hP₂.cs .r15 (by decide), hP₁.cs .r15 (by decide), h15] + refine WP.seq (WP.mono (copyR_ok hF hp (j := 0) (by decide) h₅) fun s₆ ⟨h₆, f₆⟩ => ?_) + refine WP.seq (WP.mono (copyR_ok hF hp (j := 1) (by decide) h₆) fun s₇ ⟨h₇, f₇⟩ => ?_) + refine WP.seq (WP.mono (copyR_ok hF hp (j := 2) (by decide) h₇) fun s₈ ⟨h₈, f₈⟩ => ?_) + exact WP.mono (copyR_ok hF hp (j := 3) (by decide) h₈) fun s₉ ⟨h₉, f₉⟩ => + ⟨h₉, by rw [f₉, f₈, f₇, f₆, f₅]⟩ theorem setKL_taint : ∀ v < 16, ∀ w < 16, (taint.check (X86_64.Taint.ofRegs [.rbx]) (.block (setB (sc oKL) v ++ setB (sc (oKL + 1)) w)) (.block [])).isSome = true := by decide +kernel diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Top.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Top.lean index fea8439e0..2f93be7e2 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Top.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Top.lean @@ -140,7 +140,7 @@ theorem rest_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) : theorem keyGen_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) : Piece p (fun σ s => s = σ) (fun σ s => abiPreserved σ s ∧ (kgK p).post σ s) (keyGen P p) := - (pro_piece hF).seq ((seeds_piece hF).seq ((sampA_piece hP hF).seq ((sampS_piece hP hF).seq + (pro_piece hF).seq ((seeds_piece hF).seq ((sampAll_piece hP hF).seq ((sampS_piece hP hF).seq ((rest_piece hP hF).seq (epi_piece hF))))) theorem keyGen_correct {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) (σ : State) (hp : (kgK p).pre σ) : diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs index 3ac477362..b51c4cdf8 100644 --- a/src/asm/x86_64/mldsa44.rs +++ b/src/asm/x86_64/mldsa44.rs @@ -132,119 +132,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "jne 21b", "mov eax, 0", "mov BYTE PTR [rbx+1281], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 4096", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "22:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 22b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1442", "mov rsi, rbx", - "add rsi, 5120", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 5120", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "23:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 23b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1476", "mov rsi, rbx", - "add rsi, 6144", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 6144", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "24:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1510", "mov rsi, rbx", - "add rsi, 7168", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 7168", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 4096", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 31744", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 8192", - "mov ecx, 256", + "add rdi, 4096", + "mov ecx, 1024", "26:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -252,270 +216,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 26b", + "mov eax, 0", + "mov BYTE PTR [rbx+1440], al", "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 9216", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 9216", - "mov ecx, 256", - "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 27b", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1475], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1508], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 10240", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 10240", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", + "mov BYTE PTR [rbx+1509], al", "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1542], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 31744", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 11264", - "mov ecx, 256", - "29:", + "add rdi, 8192", + "mov ecx, 1024", + "27:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 29b", + "jne 27b", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1440], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 12288", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 12288", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1474], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 13312", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 13312", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", + "mov BYTE PTR [rbx+1475], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1508], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 14336", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 14336", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", + "mov BYTE PTR [rbx+1509], al", "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1542], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 31744", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 15360", - "mov ecx, 256", - "213:", + "add rdi, 12288", + "mov ecx, 1024", + "28:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 213b", + "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1440], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 16384", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 16384", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1474], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 17408", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 17408", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", + "mov BYTE PTR [rbx+1475], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1508], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 18432", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 18432", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", + "mov BYTE PTR [rbx+1509], al", "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1542], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 16384", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 31744", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 19456", - "mov ecx, 256", - "217:", + "add rdi, 16384", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 217b", + "jne 29b", "mov eax, 0", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -532,13 +340,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 20480", "mov ecx, 256", - "218:", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 218b", + "jne 210b", "mov eax, 1", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -555,13 +363,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 21504", "mov ecx, 256", - "219:", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 219b", + "jne 211b", "mov eax, 2", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -578,13 +386,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 22528", "mov ecx, 256", - "220:", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 212b", "mov eax, 3", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -601,13 +409,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 213b", "mov eax, 4", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -624,13 +432,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 24576", "mov ecx, 256", - "222:", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", + "jne 214b", "mov eax, 5", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -647,13 +455,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 25600", "mov ecx, 256", - "223:", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", + "jne 215b", "mov eax, 6", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -670,13 +478,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 26624", "mov ecx, 256", - "224:", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", + "jne 216b", "mov eax, 7", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -693,49 +501,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 27648", "mov ecx, 256", - "225:", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", + "jne 217b", "mov rdi, r12", "add rdi, 0", "mov rsi, rbx", "add rsi, 1024", "mov ecx, 32", - "226:", + "218:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 226b", + "jne 218b", "mov rdi, r13", "add rdi, 0", "mov rsi, rbx", "add rsi, 1024", "mov ecx, 32", - "227:", + "219:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 227b", + "jne 219b", "mov rdi, r13", "add rdi, 32", "mov rsi, rbx", "add rsi, 1120", "mov ecx, 32", - "228:", + "220:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 228b", + "jne 220b", "mov rdi, rbx", "add rdi, 20480", "mov esi, 2", @@ -1125,7 +933,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32 vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, @@ -3390,119 +3198,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "jne 21b", "mov eax, 0", "mov BYTE PTR [rbx+1281], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 4096", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "22:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 22b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1442", "mov rsi, rbx", - "add rsi, 5120", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 5120", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "23:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 23b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1476", "mov rsi, rbx", - "add rsi, 6144", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 6144", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "24:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1510", "mov rsi, rbx", - "add rsi, 7168", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 7168", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 4096", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 31744", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 8192", - "mov ecx, 256", + "add rdi, 4096", + "mov ecx, 1024", "26:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -3510,270 +3282,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "add rdi, 4", "sub rcx, 1", "jne 26b", + "mov eax, 0", + "mov BYTE PTR [rbx+1440], al", "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 9216", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 9216", - "mov ecx, 256", - "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 27b", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1475], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1508], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 10240", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 10240", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", + "mov BYTE PTR [rbx+1509], al", "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1542], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 31744", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 11264", - "mov ecx, 256", - "29:", + "add rdi, 8192", + "mov ecx, 1024", + "27:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 29b", + "jne 27b", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1440], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 12288", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 12288", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1474], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 13312", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 13312", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", + "mov BYTE PTR [rbx+1475], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1508], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 14336", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 14336", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", + "mov BYTE PTR [rbx+1509], al", "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1542], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 31744", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 15360", - "mov ecx, 256", - "213:", + "add rdi, 12288", + "mov ecx, 1024", + "28:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 213b", + "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1440], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 16384", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 16384", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1474], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 17408", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 17408", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", + "mov BYTE PTR [rbx+1475], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1508], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 18432", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 18432", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", + "mov BYTE PTR [rbx+1509], al", "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1542], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 16384", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 31744", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 19456", - "mov ecx, 256", - "217:", + "add rdi, 16384", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 217b", + "jne 29b", "mov eax, 0", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -3790,13 +3406,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 20480", "mov ecx, 256", - "218:", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 218b", + "jne 210b", "mov eax, 1", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -3813,13 +3429,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 21504", "mov ecx, 256", - "219:", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 219b", + "jne 211b", "mov eax, 2", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -3836,13 +3452,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 22528", "mov ecx, 256", - "220:", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 212b", "mov eax, 3", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -3859,13 +3475,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 213b", "mov eax, 4", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -3882,13 +3498,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 24576", "mov ecx, 256", - "222:", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", + "jne 214b", "mov eax, 5", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -3905,13 +3521,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 25600", "mov ecx, 256", - "223:", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", + "jne 215b", "mov eax, 6", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -3928,13 +3544,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 26624", "mov ecx, 256", - "224:", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", + "jne 216b", "mov eax, 7", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -3951,49 +3567,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 27648", "mov ecx, 256", - "225:", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", + "jne 217b", "mov rdi, r12", "add rdi, 0", "mov rsi, rbx", "add rsi, 1024", "mov ecx, 32", - "226:", + "218:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 226b", + "jne 218b", "mov rdi, r13", "add rdi, 0", "mov rsi, rbx", "add rsi, 1024", "mov ecx, 32", - "227:", + "219:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 227b", + "jne 219b", "mov rdi, r13", "add rdi, 32", "mov rsi, rbx", "add rsi, 1120", "mov ecx, 32", - "228:", + "220:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 228b", + "jne 220b", "mov rdi, rbx", "add rdi, 20480", "mov esi, 2", @@ -4383,7 +3999,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs index f85fb71ee..7d80620cf 100644 --- a/src/asm/x86_64/mldsa65.rs +++ b/src/asm/x86_64/mldsa65.rs @@ -132,119 +132,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "jne 21b", "mov eax, 0", "mov BYTE PTR [rbx+1281], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 4096", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "22:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 22b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1442", "mov rsi, rbx", - "add rsi, 5120", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 5120", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "23:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 23b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1476", "mov rsi, rbx", - "add rsi, 6144", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 6144", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "24:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1510", "mov rsi, rbx", - "add rsi, 7168", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 7168", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 4096", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 8192", - "mov ecx, 256", + "add rdi, 4096", + "mov ecx, 1024", "26:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -252,23 +216,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 26b", + "mov eax, 4", + "mov BYTE PTR [rbx+1440], al", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1474], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 9216", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 9216", - "mov ecx, 256", + "add rdi, 8192", + "mov ecx, 1024", "27:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -276,23 +252,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 27b", + "mov eax, 3", + "mov BYTE PTR [rbx+1440], al", "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1474], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 10240", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 10240", - "mov ecx, 256", + "add rdi, 12288", + "mov ecx, 1024", "28:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -301,22 +289,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "sub rcx, 1", "jne 28b", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 16384", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 11264", - "mov ecx, 256", + "add rdi, 16384", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -324,23 +324,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 29b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 12288", + "add rsi, 20480", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 12288", - "mov ecx, 256", + "add rdi, 20480", + "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -348,23 +360,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 210b", + "mov eax, 0", + "mov BYTE PTR [rbx+1440], al", "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 13312", + "add rsi, 24576", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 13312", - "mov ecx, 256", + "add rdi, 24576", + "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -372,23 +396,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 211b", + "mov eax, 4", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1441], al", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1509], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 14336", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 14336", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -396,14 +432,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 212b", - "mov eax, 1", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -411,7 +447,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 15360", + "add rdi, 32768", "mov ecx, 256", "213:", "mov eax, DWORD PTR [rdi]", @@ -420,14 +456,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 213b", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 16384", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -435,7 +471,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 16384", + "add rdi, 33792", "mov ecx, 256", "214:", "mov eax, DWORD PTR [rdi]", @@ -444,22 +480,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 214b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 17408", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 34816", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 17408", + "add rdi, 34816", "mov ecx, 256", "215:", "mov eax, DWORD PTR [rdi]", @@ -468,22 +503,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 215b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 18432", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 35840", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 18432", + "add rdi, 35840", "mov ecx, 256", "216:", "mov eax, DWORD PTR [rdi]", @@ -492,22 +526,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 216b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 19456", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 36864", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 19456", + "add rdi, 36864", "mov ecx, 256", "217:", "mov eax, DWORD PTR [rdi]", @@ -516,22 +549,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 217b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 20480", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 37888", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 20480", + "add rdi, 37888", "mov ecx, 256", "218:", "mov eax, DWORD PTR [rdi]", @@ -540,22 +572,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 218b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 21504", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 38912", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 21504", + "add rdi, 38912", "mov ecx, 256", "219:", "mov eax, DWORD PTR [rdi]", @@ -564,22 +595,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 219b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 22528", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 39936", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 22528", + "add rdi, 39936", "mov ecx, 256", "220:", "mov eax, DWORD PTR [rdi]", @@ -588,22 +618,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 220b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 23552", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 40960", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 23552", + "add rdi, 40960", "mov ecx, 256", "221:", "mov eax, DWORD PTR [rdi]", @@ -612,22 +641,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 221b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 24576", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 41984", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", + "add rdi, 41984", "mov ecx, 256", "222:", "mov eax, DWORD PTR [rdi]", @@ -636,22 +664,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 222b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 8", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 25600", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 43008", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 43008", "mov ecx, 256", "223:", "mov eax, DWORD PTR [rdi]", @@ -660,22 +687,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 223b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 9", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 26624", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 44032", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 44032", "mov ecx, 256", "224:", "mov eax, DWORD PTR [rdi]", @@ -684,22 +710,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 224b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 10", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 27648", + "add rdi, 1216", + "mov esi, 4", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 45056", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 27648", + "add rdi, 45056", "mov ecx, 256", "225:", "mov eax, DWORD PTR [rdi]", @@ -708,489 +733,92 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 225b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "mov rdi, r12", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 226b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "mov rdi, r13", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 227b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "mov rdi, r13", + "add rdi, 32", "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rsi, 1120", + "mov ecx, 32", "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 228b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 33792", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 33792", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 0", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 4", - "mov rdx, rbx", - "add rdx, 34816", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", "mov rdi, rbx", "add rdi, 34816", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 1", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", "mov esi, 4", - "mov rdx, rbx", - "add rdx, 35840", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 128", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 35840", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 2", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", "mov esi, 4", - "mov rdx, rbx", - "add rdx, 36864", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 256", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 36864", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov eax, 3", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", "mov esi, 4", - "mov rdx, rbx", - "add rdx, 37888", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 384", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 37888", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov eax, 4", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", "mov esi, 4", - "mov rdx, rbx", - "add rdx, 38912", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 512", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 38912", - "mov ecx, 256", - "236:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 236b", - "mov eax, 5", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", "mov esi, 4", - "mov rdx, rbx", - "add rdx, 39936", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 640", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 39936", - "mov ecx, 256", - "237:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 237b", - "mov eax, 6", - "mov BYTE PTR [rbx+1280], al", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 768", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 4", - "mov rdx, rbx", - "add rdx, 40960", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", - "238:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 238b", - "mov eax, 7", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 4", - "mov rdx, rbx", - "add rdx, 41984", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 41984", - "mov ecx, 256", - "239:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 239b", - "mov eax, 8", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 4", - "mov rdx, rbx", - "add rdx, 43008", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 43008", - "mov ecx, 256", - "240:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 240b", - "mov eax, 9", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 4", - "mov rdx, rbx", - "add rdx, 44032", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 44032", - "mov ecx, 256", - "241:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 241b", - "mov eax, 10", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 4", - "mov rdx, rbx", - "add rdx, 45056", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "242:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 242b", - "mov rdi, r12", - "add rdi, 0", - "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "243:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 243b", - "mov rdi, r13", - "add rdi, 0", - "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "244:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 244b", - "mov rdi, r13", - "add rdi, 32", - "mov rsi, rbx", - "add rsi, 1120", - "mov ecx, 32", - "245:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 245b", - "mov rdi, rbx", - "add rdi, 34816", - "mov esi, 4", - "mov edx, 4", - "mov rcx, r13", - "add rcx, 128", - "mov r8d, 128", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 35840", - "mov esi, 4", - "mov edx, 4", - "mov rcx, r13", - "add rcx, 256", - "mov r8d, 128", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 36864", - "mov esi, 4", - "mov edx, 4", - "mov rcx, r13", - "add rcx, 384", - "mov r8d, 128", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 37888", - "mov esi, 4", - "mov edx, 4", - "mov rcx, r13", - "add rcx, 512", - "mov r8d, 128", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 38912", - "mov esi, 4", - "mov edx, 4", - "mov rcx, r13", - "add rcx, 640", - "mov r8d, 128", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 39936", - "mov esi, 4", - "mov edx, 4", - "mov rcx, r13", - "add rcx, 768", - "mov r8d, 128", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 40960", + "add rdi, 40960", "mov esi, 4", "mov edx, 4", "mov rcx, r13", @@ -1721,6 +1349,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32 vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, @@ -4986,678 +4615,306 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "jne 21b", "mov eax, 0", "mov BYTE PTR [rbx+1281], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 4096", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "22:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 22b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1442", "mov rsi, rbx", - "add rsi, 5120", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 5120", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "23:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 23b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1476", "mov rsi, rbx", - "add rsi, 6144", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 6144", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "24:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 24b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 7168", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 7168", - "mov ecx, 256", - "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 25b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 8192", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 8192", - "mov ecx, 256", - "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 26b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 9216", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 9216", - "mov ecx, 256", - "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 27b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 10240", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 10240", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 11264", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 11264", - "mov ecx, 256", - "29:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 29b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 12288", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 12288", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 13312", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 13312", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 14336", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 14336", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 15360", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 15360", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 16384", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 16384", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 17408", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 17408", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 18432", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 18432", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 19456", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 19456", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 20480", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 20480", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 21504", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 21504", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 22528", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 22528", - "mov ecx, 256", - "220:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 220b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", + "jne 24b", + "mov rdi, rbx", + "add rdi, 1510", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "25:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 25b", + "mov eax, 0", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1509], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 4096", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 23552", - "mov ecx, 256", - "221:", + "add rdi, 4096", + "mov ecx, 1024", + "26:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "jne 26b", "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 24576", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", - "mov ecx, 256", - "222:", + "add rdi, 8192", + "mov ecx, 1024", + "27:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", + "jne 27b", + "mov eax, 3", + "mov BYTE PTR [rbx+1440], al", "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 25600", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 25600", - "mov ecx, 256", - "223:", + "add rdi, 12288", + "mov ecx, 1024", + "28:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", + "jne 28b", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1475], al", "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 26624", + "add rsi, 16384", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov ecx, 256", - "224:", + "add rdi, 16384", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", + "jne 29b", + "mov eax, 1", + "mov BYTE PTR [rbx+1440], al", "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1509], al", "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 27648", + "add rsi, 20480", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 27648", - "mov ecx, 256", - "225:", + "add rdi, 20480", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", + "jne 210b", + "mov eax, 0", + "mov BYTE PTR [rbx+1440], al", "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1474], al", "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 28672", + "add rsi, 24576", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", - "226:", + "add rdi, 24576", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 226b", + "jne 211b", + "mov eax, 4", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1441], al", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1474], al", "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", + "mov BYTE PTR [rbx+1475], al", "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1508], al", "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", + "mov BYTE PTR [rbx+1509], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1542], al", "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 49152", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", - "229:", + "add rdi, 28672", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 229b", + "jne 212b", "mov eax, 3", "mov BYTE PTR [rbx+1184], al", "mov eax, 5", @@ -5675,13 +4932,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 32768", "mov ecx, 256", - "230:", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 230b", + "jne 213b", "mov eax, 4", "mov BYTE PTR [rbx+1184], al", "mov eax, 5", @@ -5699,13 +4956,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 33792", "mov ecx, 256", - "231:", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 231b", + "jne 214b", "mov eax, 0", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5722,13 +4979,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 34816", "mov ecx, 256", - "232:", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 232b", + "jne 215b", "mov eax, 1", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5745,13 +5002,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 35840", "mov ecx, 256", - "233:", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 233b", + "jne 216b", "mov eax, 2", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5768,13 +5025,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 36864", "mov ecx, 256", - "234:", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 234b", + "jne 217b", "mov eax, 3", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5791,13 +5048,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 37888", "mov ecx, 256", - "235:", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 235b", + "jne 218b", "mov eax, 4", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5814,13 +5071,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 38912", "mov ecx, 256", - "236:", + "219:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 236b", + "jne 219b", "mov eax, 5", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5837,13 +5094,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 39936", "mov ecx, 256", - "237:", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 237b", + "jne 220b", "mov eax, 6", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5860,13 +5117,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 40960", "mov ecx, 256", - "238:", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 238b", + "jne 221b", "mov eax, 7", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5883,13 +5140,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 41984", "mov ecx, 256", - "239:", + "222:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 239b", + "jne 222b", "mov eax, 8", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5906,13 +5163,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 43008", "mov ecx, 256", - "240:", + "223:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 240b", + "jne 223b", "mov eax, 9", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5929,13 +5186,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 44032", "mov ecx, 256", - "241:", + "224:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 241b", + "jne 224b", "mov eax, 10", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -5952,49 +5209,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 45056", "mov ecx, 256", - "242:", + "225:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 242b", + "jne 225b", "mov rdi, r12", "add rdi, 0", "mov rsi, rbx", "add rsi, 1024", "mov ecx, 32", - "243:", + "226:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 243b", + "jne 226b", "mov rdi, r13", "add rdi, 0", "mov rsi, rbx", "add rsi, 1024", "mov ecx, 32", - "244:", + "227:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 244b", + "jne 227b", "mov rdi, r13", "add rdi, 32", "mov rsi, rbx", "add rsi, 1120", "mov ecx, 32", - "245:", + "228:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 245b", + "jne 228b", "mov rdi, rbx", "add rdi, 34816", "mov esi, 4", @@ -6575,6 +5832,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs index 50336abce..f8dfeb95d 100644 --- a/src/asm/x86_64/mldsa87.rs +++ b/src/asm/x86_64/mldsa87.rs @@ -132,119 +132,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "jne 21b", "mov eax, 0", "mov BYTE PTR [rbx+1281], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 4096", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "22:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 22b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1442", "mov rsi, rbx", - "add rsi, 5120", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 5120", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "23:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 23b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1476", "mov rsi, rbx", - "add rsi, 6144", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 6144", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "24:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1510", "mov rsi, rbx", - "add rsi, 7168", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 7168", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 4096", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 8192", - "mov ecx, 256", + "add rdi, 4096", + "mov ecx, 1024", "26:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -252,23 +216,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 26b", + "mov eax, 4", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1441], al", "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1474], al", "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 9216", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 9216", - "mov ecx, 256", + "add rdi, 8192", + "mov ecx, 1024", "27:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -276,23 +252,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 27b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 10240", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 10240", - "mov ecx, 256", + "add rdi, 12288", + "mov ecx, 1024", "28:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -300,23 +288,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 28b", + "mov eax, 5", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1475], al", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1509], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 16384", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 11264", - "mov ecx, 256", + "add rdi, 16384", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -324,23 +324,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 29b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 12288", + "add rsi, 20480", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 12288", - "mov ecx, 256", + "add rdi, 20480", + "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -348,23 +360,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 210b", + "mov eax, 6", + "mov BYTE PTR [rbx+1440], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1475], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 13312", + "add rsi, 24576", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 13312", - "mov ecx, 256", + "add rdi, 24576", + "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -373,22 +397,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "sub rcx, 1", "jne 211b", "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 14336", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 14336", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -396,23 +432,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+1440], al", "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1441], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 32768", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 15360", - "mov ecx, 256", + "add rdi, 32768", + "mov ecx, 1024", "213:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -420,23 +468,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 213b", + "mov eax, 4", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1441], al", "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 16384", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 16384", - "mov ecx, 256", + "add rdi, 36864", + "mov ecx, 1024", "214:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -444,23 +504,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 214b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 17408", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1543], al", + "mov rdi, rbx", + "add rdi, 1408", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 17408", - "mov ecx, 256", + "add rdi, 40960", + "mov ecx, 1024", "215:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -468,23 +540,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 215b", + "mov eax, 5", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1475], al", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 18432", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 18432", - "mov ecx, 256", + "add rdi, 45056", + "mov ecx, 1024", "216:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -492,23 +576,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 216b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 49152", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 19456", - "mov ecx, 256", + "add rdi, 49152", + "mov ecx, 1024", "217:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -516,23 +612,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 217b", + "mov eax, 6", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1509], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 20480", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 20480", - "mov ecx, 256", + "add rdi, 53248", + "mov ecx, 1024", "218:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -541,22 +649,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "sub rcx, 1", "jne 218b", "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 21504", + "add rsi, 57344", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 21504", - "mov ecx, 256", + "add rdi, 57344", + "mov ecx, 1024", "219:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", @@ -564,22 +684,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 219b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 22528", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 22528", + "add rdi, 61440", "mov ecx, 256", "220:", "mov eax, DWORD PTR [rdi]", @@ -588,22 +707,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 220b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 23552", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 62464", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 23552", + "add rdi, 62464", "mov ecx, 256", "221:", "mov eax, DWORD PTR [rdi]", @@ -612,22 +730,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 221b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 24576", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 63488", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", + "add rdi, 63488", "mov ecx, 256", "222:", "mov eax, DWORD PTR [rdi]", @@ -636,22 +753,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 222b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 25600", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 64512", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 64512", "mov ecx, 256", "223:", "mov eax, DWORD PTR [rdi]", @@ -660,22 +776,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 223b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 26624", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 65536", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 65536", "mov ecx, 256", "224:", "mov eax, DWORD PTR [rdi]", @@ -684,22 +799,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 224b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 27648", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 66560", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 27648", + "add rdi, 66560", "mov ecx, 256", "225:", "mov eax, DWORD PTR [rdi]", @@ -708,22 +822,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 225b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 28672", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 67584", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 28672", + "add rdi, 67584", "mov ecx, 256", "226:", "mov eax, DWORD PTR [rdi]", @@ -732,22 +845,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 226b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 29696", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 68608", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 29696", + "add rdi, 68608", "mov ecx, 256", "227:", "mov eax, DWORD PTR [rdi]", @@ -756,22 +868,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 227b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 8", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 30720", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 69632", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 30720", + "add rdi, 69632", "mov ecx, 256", "228:", "mov eax, DWORD PTR [rdi]", @@ -780,22 +891,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 228b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 9", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 31744", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 70656", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 31744", + "add rdi, 70656", "mov ecx, 256", "229:", "mov eax, DWORD PTR [rdi]", @@ -804,22 +914,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 229b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 10", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 32768", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 71680", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 32768", + "add rdi, 71680", "mov ecx, 256", "230:", "mov eax, DWORD PTR [rdi]", @@ -828,22 +937,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 230b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 11", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 33792", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 72704", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 72704", "mov ecx, 256", "231:", "mov eax, DWORD PTR [rdi]", @@ -852,22 +960,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 231b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 12", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 34816", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 73728", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 73728", "mov ecx, 256", "232:", "mov eax, DWORD PTR [rdi]", @@ -876,22 +983,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 232b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 13", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 35840", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 74752", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 74752", "mov ecx, 256", "233:", "mov eax, DWORD PTR [rdi]", @@ -900,22 +1006,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 233b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "mov eax, 14", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 36864", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 75776", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 36864", + "add rdi, 75776", "mov ecx, 256", "234:", "mov eax, DWORD PTR [rdi]", @@ -924,1140 +1029,648 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 4", "sub rcx, 1", "jne 234b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "mov rdi, r12", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 235b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "mov rdi, r13", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", + "add rsi, 1024", + "mov ecx, 32", "236:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 236b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "mov rdi, r13", + "add rdi, 32", "mov rsi, rbx", - "add rsi, 39936", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", + "add rsi, 1120", + "mov ecx, 32", "237:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 237b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 40960", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", - "238:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 238b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 61440", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 128", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 41984", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 62464", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 224", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 41984", - "mov ecx, 256", - "239:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 239b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 63488", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 320", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 43008", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 64512", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 416", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 43008", - "mov ecx, 256", - "240:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 240b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 65536", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 512", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 44032", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 66560", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 608", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 44032", - "mov ecx, 256", - "241:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 241b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 67584", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 704", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 68608", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 800", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 69632", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 896", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 70656", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 992", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 71680", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1088", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 72704", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1184", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 73728", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1280", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 74752", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1376", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 75776", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1472", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 61440", "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "242:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 242b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 62464", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 63488", "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "243:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 243b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 64512", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 65536", "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "244:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 244b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 66560", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 67584", "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "245:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 245b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 4096", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 5120", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", - "246:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 246b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 6144", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 7168", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 50176", - "mov ecx, 256", - "247:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 247b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 8192", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 51200", + "add rsi, 9216", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 51200", - "mov ecx, 256", - "248:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 248b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 52224", + "add rsi, 10240", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 52224", - "mov ecx, 256", - "249:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 249b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 68608", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "250:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 250b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 32", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1568", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 11264", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "251:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 251b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 12288", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 13312", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "252:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 252b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 14336", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 15360", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "253:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 253b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 17408", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "254:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 254b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 69632", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "255:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 255b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 352", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1984", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 18432", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "256:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 256b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 19456", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 60416", + "add rsi, 20480", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 60416", - "mov ecx, 256", - "257:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 257b", - "mov eax, 0", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 21504", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 22528", "mov rdx, rbx", - "add rdx, 61440", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "258:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 258b", - "mov eax, 1", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 24576", "mov rdx, rbx", - "add rdx, 62464", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "259:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 259b", - "mov eax, 2", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 70656", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 63488", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "260:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 260b", - "mov eax, 3", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 672", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 2400", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 25600", "mov rdx, rbx", - "add rdx, 64512", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "261:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 261b", - "mov eax, 4", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 26624", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 27648", "mov rdx, rbx", - "add rdx, 65536", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "262:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 262b", - "mov eax, 5", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 29696", "mov rdx, rbx", - "add rdx, 66560", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 66560", - "mov ecx, 256", - "263:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 263b", - "mov eax, 6", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 31744", "mov rdx, rbx", "add rdx, 67584", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 67584", - "mov ecx, 256", - "264:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 264b", - "mov eax, 7", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 71680", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 68608", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 68608", - "mov ecx, 256", - "265:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 265b", - "mov eax, 8", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 992", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 2816", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 32768", "mov rdx, rbx", - "add rdx, 69632", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "266:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 266b", - "mov eax, 9", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 34816", "mov rdx, rbx", - "add rdx, 70656", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "267:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 267b", - "mov eax, 10", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 35840", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 71680", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "268:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 268b", - "mov eax, 11", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 38912", "mov rdx, rbx", - "add rdx, 72704", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "269:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 269b", - "mov eax, 12", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 72704", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 73728", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "270:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 270b", - "mov eax, 13", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 1312", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 3232", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 39936", "mov rdx, rbx", - "add rdx, 74752", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "271:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 271b", - "mov eax, 14", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 41984", "mov rdx, rbx", - "add rdx, 75776", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "272:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 272b", - "mov rdi, r12", - "add rdi, 0", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "273:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 273b", - "mov rdi, r13", - "add rdi, 0", + "add rsi, 43008", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "274:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 274b", - "mov rdi, r13", - "add rdi, 32", + "add rsi, 44032", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1120", - "mov ecx, 32", - "275:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 275b", - "mov rdi, rbx", - "add rdi, 61440", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 128", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 62464", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 224", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 63488", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 320", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 64512", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 416", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 65536", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 512", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 66560", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 608", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 67584", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 704", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 68608", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 800", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 69632", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 896", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 70656", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 992", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 71680", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1088", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 72704", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1184", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 73728", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1280", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 74752", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1376", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 75776", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1472", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 61440", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 62464", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 63488", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 64512", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 65536", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 66560", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 67584", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 5120", - "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 6144", - "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 7168", - "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 8192", - "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 9216", - "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 10240", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -2069,7 +1682,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 68608", + "add rsi, 73728", "call {vg_mldsa_add_avx2}", "mov rdi, rbx", "add rdi, 76800", @@ -2082,7 +1695,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 32", + "add rdx, 1632", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -2090,55 +1703,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 1568", + "add rcx, 3648", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 12288", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 13312", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 14336", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 16384", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 17408", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -2150,7 +1763,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 74752", "call {vg_mldsa_add_avx2}", "mov rdi, rbx", "add rdi, 76800", @@ -2163,7 +1776,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 352", + "add rdx, 1952", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -2171,55 +1784,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 1984", + "add rcx, 4064", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 18432", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 20480", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 21504", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 22528", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 59392", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 24576", + "add rsi, 60416", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -2231,7 +1844,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 75776", "call {vg_mldsa_add_avx2}", "mov rdi, rbx", "add rdi, 76800", @@ -2244,7 +1857,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 672", + "add rdx, 2272", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -2252,2297 +1865,1892 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32 "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 2400", + "add rcx, 4480", "mov r8d, 416", "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 25600", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 26624", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 2592", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 8", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r13", + "add rcx, 64", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, + vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, + vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, + vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + ) +} + +/// The CPU features `vg_mldsa87_sign_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA87_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-87 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address. +/// +/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing. +/// +/// # Safety +/// +/// * `sk` must be valid for reads of 4896 bytes. +/// * `mu` must be valid for reads of 64 bytes. +/// * `rnd` must be valid for reads of 32 bytes. +/// * `sig` must be valid for reads and writes of 4627 bytes. +/// * `scratch` must be valid for reads and writes of 144384 bytes. +/// * `sk` must have been written by `vg_mldsa87_keygen`. +/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do). +/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [r8+840], rbx", + "mov QWORD PTR [r8+848], rbp", + "mov QWORD PTR [r8+856], r12", + "mov QWORD PTR [r8+864], r13", + "mov QWORD PTR [r8+872], r14", + "mov QWORD PTR [r8+880], r15", + "mov rbx, r8", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r14, rcx", + "mov r15d, 1", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 64512", "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 27648", + "add rsi, 65536", "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 28672", + "add rsi, 66560", "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 29696", + "add rsi, 67584", "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 30720", + "add rsi, 68608", "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt_avx2}", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", "add rsi, 71680", - "call {vg_mldsa_add_avx2}", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 72704", "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 992", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 2816", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 73728", "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 74752", "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 75776", "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 35840", + "add rsi, 76800", "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 78848", "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 79872", "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt_avx2}", + "add rsi, 80896", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 72704", - "call {vg_mldsa_add_avx2}", + "add rsi, 81920", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 82944", "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 1312", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 3232", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 83968", "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 84992", "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 86016", "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 87040", "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 44032", + "add rsi, 88064", "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 89088", "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 90112", "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt_avx2}", + "add rsi, 91136", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 73728", - "call {vg_mldsa_add_avx2}", + "add rsi, 92160", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 93184", "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 1632", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 3648", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 94208", "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 95232", "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 96256", "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 97280", "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 51200", + "add rsi, 98304", "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 52224", + "add rsi, 99328", "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 100352", "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt_avx2}", + "add rsi, 101376", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 74752", - "call {vg_mldsa_add_avx2}", + "add rsi, 102400", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 103424", "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 1952", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 4064", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 104448", "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 105472", "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 106496", "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 107520", "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 108544", "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", - "add rdi, 76800", + "add rdi, 912", "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 60416", - "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 75776", - "call {vg_mldsa_add_avx2}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 2272", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 4480", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, r12", - "add rcx, 0", - "mov r8d, 2592", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 8", - "mov ecx, 31", - "mov r8, rbx", - "add r8, 200", - "call {vg_keccak_pad}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, r13", - "add rcx, 64", - "mov r8d, 64", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_squeeze}", - "mov eax, r15d", - "mov r15, QWORD PTR [rbx+880]", - "mov r14, QWORD PTR [rbx+872]", - "mov r13, QWORD PTR [rbx+864]", - "mov r12, QWORD PTR [rbx+856]", - "mov rbp, QWORD PTR [rbx+848]", - "mov rbx, QWORD PTR [rbx+840]", - "ret", - vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, - vg_keccak_pad = sym super::sha3::vg_keccak_pad, - vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, - vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, - vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, - vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, - vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, - vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, - vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, - vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, - vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round, - vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, - ) -} - -/// The CPU features `vg_mldsa87_sign_avx2` requires (`Artifact.features`). -pub(crate) const VG_MLDSA87_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; - -/// ML-DSA-87 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. -/// -/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness. -/// -/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address. -/// -/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing. -/// -/// # Safety -/// -/// * `sk` must be valid for reads of 4896 bytes. -/// * `mu` must be valid for reads of 64 bytes. -/// * `rnd` must be valid for reads of 32 bytes. -/// * `sig` must be valid for reads and writes of 4627 bytes. -/// * `scratch` must be valid for reads and writes of 144384 bytes. -/// * `sk` must have been written by `vg_mldsa87_keygen`. -/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing. -/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). -/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do). -/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). -/// * The CPU must support the `avx` and `avx2` target features. -#[unsafe(naked)] -pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { - core::arch::naked_asm!( - "mov QWORD PTR [r8+840], rbx", - "mov QWORD PTR [r8+848], rbp", - "mov QWORD PTR [r8+856], r12", - "mov QWORD PTR [r8+864], r13", - "mov QWORD PTR [r8+872], r14", - "mov QWORD PTR [r8+880], r15", - "mov rbx, r8", - "mov rbp, rdi", - "mov r12, rsi", - "mov r13, rdx", - "mov r14, rcx", - "mov r15d, 1", - "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbp", - "add rsi, 0", - "mov ecx, 32", - "20:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 20b", - "mov eax, 0", - "mov BYTE PTR [rbx+944], al", - "mov eax, 0", - "mov BYTE PTR [rbx+945], al", - "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 64512", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 1", - "mov BYTE PTR [rbx+944], al", - "mov eax, 0", - "mov BYTE PTR [rbx+945], al", - "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 65536", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 2", - "mov BYTE PTR [rbx+944], al", - "mov eax, 0", - "mov BYTE PTR [rbx+945], al", - "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 66560", + "add rsi, 109568", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 3", "mov BYTE PTR [rbx+944], al", - "mov eax, 0", + "mov eax, 6", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 67584", + "add rsi, 110592", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 4", "mov BYTE PTR [rbx+944], al", - "mov eax, 0", + "mov eax, 6", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 68608", + "add rsi, 111616", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 5", "mov BYTE PTR [rbx+944], al", - "mov eax, 0", + "mov eax, 6", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 112640", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 6", "mov BYTE PTR [rbx+944], al", - "mov eax, 0", + "mov eax, 6", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 113664", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 0", "mov BYTE PTR [rbx+944], al", - "mov eax, 1", + "mov eax, 7", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 114688", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 1", "mov BYTE PTR [rbx+944], al", - "mov eax, 1", + "mov eax, 7", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 115712", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 2", "mov BYTE PTR [rbx+944], al", - "mov eax, 1", + "mov eax, 7", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 116736", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 3", "mov BYTE PTR [rbx+944], al", - "mov eax, 1", + "mov eax, 7", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 117760", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 4", "mov BYTE PTR [rbx+944], al", - "mov eax, 1", + "mov eax, 7", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 118784", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 5", "mov BYTE PTR [rbx+944], al", - "mov eax, 1", + "mov eax, 7", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 76800", + "add rsi, 119808", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", "mov eax, 6", "mov BYTE PTR [rbx+944], al", - "mov eax, 1", + "mov eax, 7", "mov BYTE PTR [rbx+945], al", "mov rdi, rbx", "add rdi, 912", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 120832", "mov rdx, rbx", "add rdx, 3072", "call {vg_mldsa_rej_ntt_poly}", "and r15d, eax", - "mov eax, 0", - "mov BYTE PTR [rbx+944], al", - "mov eax, 2", - "mov BYTE PTR [rbx+945], al", - "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 78848", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 1", - "mov BYTE PTR [rbx+944], al", - "mov eax, 2", - "mov BYTE PTR [rbx+945], al", + "test r15d, r15d", + "jne 21f", + "jmp 22f", + "21:", + "mov rdi, rbp", + "add rdi, 128", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 40960", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 40960", "mov rsi, rbx", - "add rsi, 79872", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 2", - "mov BYTE PTR [rbx+944], al", - "mov eax, 2", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 224", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 41984", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 41984", "mov rsi, rbx", - "add rsi, 80896", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 3", - "mov BYTE PTR [rbx+944], al", - "mov eax, 2", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 320", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 43008", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 43008", "mov rsi, rbx", - "add rsi, 81920", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 4", - "mov BYTE PTR [rbx+944], al", - "mov eax, 2", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 416", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 44032", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 44032", "mov rsi, rbx", - "add rsi, 82944", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 5", - "mov BYTE PTR [rbx+944], al", - "mov eax, 2", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 512", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 45056", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 45056", "mov rsi, rbx", - "add rsi, 83968", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 6", - "mov BYTE PTR [rbx+944], al", - "mov eax, 2", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 608", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 46080", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 46080", "mov rsi, rbx", - "add rsi, 84992", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 0", - "mov BYTE PTR [rbx+944], al", - "mov eax, 3", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 704", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 47104", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 47104", "mov rsi, rbx", - "add rsi, 86016", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 1", - "mov BYTE PTR [rbx+944], al", - "mov eax, 3", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 800", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 48128", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 48128", "mov rsi, rbx", - "add rsi, 87040", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 2", - "mov BYTE PTR [rbx+944], al", - "mov eax, 3", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 896", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 49152", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 49152", "mov rsi, rbx", - "add rsi, 88064", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 3", - "mov BYTE PTR [rbx+944], al", - "mov eax, 3", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 992", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 50176", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 50176", "mov rsi, rbx", - "add rsi, 89088", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 4", - "mov BYTE PTR [rbx+944], al", - "mov eax, 3", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1088", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 51200", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 51200", "mov rsi, rbx", - "add rsi, 90112", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 5", - "mov BYTE PTR [rbx+944], al", - "mov eax, 3", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1184", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 52224", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 52224", "mov rsi, rbx", - "add rsi, 91136", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 6", - "mov BYTE PTR [rbx+944], al", - "mov eax, 3", - "mov BYTE PTR [rbx+945], al", - "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 92160", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 0", - "mov BYTE PTR [rbx+944], al", - "mov eax, 4", - "mov BYTE PTR [rbx+945], al", - "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 93184", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 1", - "mov BYTE PTR [rbx+944], al", - "mov eax, 4", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1280", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 53248", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 53248", "mov rsi, rbx", - "add rsi, 94208", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 2", - "mov BYTE PTR [rbx+944], al", - "mov eax, 4", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1376", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 54272", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 54272", "mov rsi, rbx", - "add rsi, 95232", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 3", - "mov BYTE PTR [rbx+944], al", - "mov eax, 4", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1472", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 55296", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 55296", "mov rsi, rbx", - "add rsi, 96256", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 4", - "mov BYTE PTR [rbx+944], al", - "mov eax, 4", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1568", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 56320", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 56320", "mov rsi, rbx", - "add rsi, 97280", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 5", - "mov BYTE PTR [rbx+944], al", - "mov eax, 4", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1984", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 57344", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 57344", "mov rsi, rbx", - "add rsi, 98304", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 6", - "mov BYTE PTR [rbx+944], al", - "mov eax, 4", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 2400", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 58368", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 58368", "mov rsi, rbx", - "add rsi, 99328", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 0", - "mov BYTE PTR [rbx+944], al", - "mov eax, 5", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 2816", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 59392", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 59392", "mov rsi, rbx", - "add rsi, 100352", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 1", - "mov BYTE PTR [rbx+944], al", - "mov eax, 5", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 3232", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 60416", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 60416", "mov rsi, rbx", - "add rsi, 101376", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 2", - "mov BYTE PTR [rbx+944], al", - "mov eax, 5", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 3648", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 61440", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 61440", "mov rsi, rbx", - "add rsi, 102400", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 3", - "mov BYTE PTR [rbx+944], al", - "mov eax, 5", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 4064", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 62464", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 62464", "mov rsi, rbx", - "add rsi, 103424", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 4", - "mov BYTE PTR [rbx+944], al", - "mov eax, 5", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 4480", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 63488", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 63488", "mov rsi, rbx", - "add rsi, 104448", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 5", - "mov BYTE PTR [rbx+944], al", - "mov eax, 5", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 105472", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 6", - "mov BYTE PTR [rbx+944], al", - "mov eax, 5", - "mov BYTE PTR [rbx+945], al", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 32", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 106496", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, r13", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 128", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 960", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", "mov eax, 0", - "mov BYTE PTR [rbx+944], al", - "mov eax, 6", - "mov BYTE PTR [rbx+945], al", + "mov QWORD PTR [rbx+896], rax", + "mov eax, 814", + "mov QWORD PTR [rbx+888], rax", + "23:", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 0", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 107520", + "add rdi, 960", + "mov esi, 524288", "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 1", - "mov BYTE PTR [rbx+944], al", - "mov eax, 6", - "mov BYTE PTR [rbx+945], al", + "add rdx, 18432", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 108544", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 2", - "mov BYTE PTR [rbx+944], al", - "mov eax, 6", - "mov BYTE PTR [rbx+945], al", + "add rsi, 18432", + "mov ecx, 1024", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 24b", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 109568", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 3", - "mov BYTE PTR [rbx+944], al", - "mov eax, 6", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 110592", + "add rdi, 960", + "mov esi, 524288", "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 4", - "mov BYTE PTR [rbx+944], al", - "mov eax, 6", - "mov BYTE PTR [rbx+945], al", + "add rdx, 19456", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 111616", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 5", - "mov BYTE PTR [rbx+944], al", - "mov eax, 6", - "mov BYTE PTR [rbx+945], al", + "add rsi, 19456", + "mov ecx, 1024", + "25:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 25b", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 112640", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 6", - "mov BYTE PTR [rbx+944], al", - "mov eax, 6", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 113664", + "add rdi, 960", + "mov esi, 524288", "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 0", - "mov BYTE PTR [rbx+944], al", - "mov eax, 7", - "mov BYTE PTR [rbx+945], al", + "add rdx, 20480", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 27648", "mov rsi, rbx", - "add rsi, 114688", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 1", - "mov BYTE PTR [rbx+944], al", - "mov eax, 7", - "mov BYTE PTR [rbx+945], al", + "add rsi, 20480", + "mov ecx, 1024", + "26:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 26b", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 27648", "mov rsi, rbx", - "add rsi, 115712", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 2", - "mov BYTE PTR [rbx+944], al", - "mov eax, 7", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 116736", + "add rdi, 960", + "mov esi, 524288", "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 3", - "mov BYTE PTR [rbx+944], al", - "mov eax, 7", - "mov BYTE PTR [rbx+945], al", + "add rdx, 21504", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 28672", "mov rsi, rbx", - "add rsi, 117760", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 4", - "mov BYTE PTR [rbx+944], al", - "mov eax, 7", - "mov BYTE PTR [rbx+945], al", + "add rsi, 21504", + "mov ecx, 1024", + "27:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 27b", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 28672", "mov rsi, rbx", - "add rsi, 118784", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 5", - "mov BYTE PTR [rbx+944], al", - "mov eax, 7", - "mov BYTE PTR [rbx+945], al", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 4", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", "mov rdi, rbx", - "add rdi, 912", - "mov rsi, rbx", - "add rsi, 119808", + "add rdi, 960", + "mov esi, 524288", "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov eax, 6", - "mov BYTE PTR [rbx+944], al", - "mov eax, 7", - "mov BYTE PTR [rbx+945], al", + "add rdx, 22528", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", - "add rdi, 912", + "add rdi, 29696", "mov rsi, rbx", - "add rsi, 120832", - "mov rdx, rbx", - "add rdx, 3072", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "test r15d, r15d", - "jne 21f", - "jmp 22f", - "21:", - "mov rdi, rbp", - "add rdi, 128", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 40960", - "call {vg_mldsa_bit_unpack}", + "add rsi, 22528", + "mov ecx, 1024", + "28:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 28b", "mov rdi, rbx", - "add rdi, 40960", + "add rdi, 29696", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 224", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 41984", - "call {vg_mldsa_bit_unpack}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 5", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", "mov rdi, rbx", - "add rdi, 41984", - "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 320", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 43008", - "call {vg_mldsa_bit_unpack}", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 23552", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", - "add rdi, 43008", + "add rdi, 30720", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 416", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 44032", - "call {vg_mldsa_bit_unpack}", + "add rsi, 23552", + "mov ecx, 1024", + "29:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 29b", "mov rdi, rbx", - "add rdi, 44032", + "add rdi, 30720", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 512", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 45056", - "call {vg_mldsa_bit_unpack}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 6", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", "mov rdi, rbx", - "add rdi, 45056", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 24576", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 31744", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 608", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 46080", - "call {vg_mldsa_bit_unpack}", + "add rsi, 24576", + "mov ecx, 1024", + "210:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 210b", "mov rdi, rbx", - "add rdi, 46080", + "add rdi, 31744", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 704", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 47104", - "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 47104", + "add rdi, 32768", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 800", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 48128", - "call {vg_mldsa_bit_unpack}", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 48128", + "add rdi, 32768", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 896", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 49152", - "call {vg_mldsa_bit_unpack}", + "add rsi, 65536", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 49152", + "add rdi, 32768", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 992", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 50176", - "call {vg_mldsa_bit_unpack}", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 50176", + "add rdi, 32768", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1088", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 51200", - "call {vg_mldsa_bit_unpack}", + "add rsi, 67584", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 51200", + "add rdi, 32768", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1184", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 52224", - "call {vg_mldsa_bit_unpack}", + "add rsi, 68608", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 52224", + "add rdi, 32768", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1280", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 53248", - "call {vg_mldsa_bit_unpack}", + "add rsi, 69632", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 53248", + "add rdi, 32768", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1376", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 54272", - "call {vg_mldsa_bit_unpack}", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 54272", + "add rdi, 32768", "mov rsi, rbx", "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1472", - "mov esi, 96", - "mov edx, 2", - "mov ecx, 2", - "mov r8, rbx", - "add r8, 55296", - "call {vg_mldsa_bit_unpack}", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 55296", + "add rdi, 33792", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1568", - "mov esi, 416", - "mov edx, 4095", - "mov ecx, 4096", - "mov r8, rbx", - "add r8, 56320", - "call {vg_mldsa_bit_unpack}", + "add rsi, 71680", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 56320", + "add rdi, 33792", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1984", - "mov esi, 416", - "mov edx, 4095", - "mov ecx, 4096", - "mov r8, rbx", - "add r8, 57344", - "call {vg_mldsa_bit_unpack}", + "add rsi, 72704", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 57344", + "add rdi, 33792", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 2400", - "mov esi, 416", - "mov edx, 4095", - "mov ecx, 4096", - "mov r8, rbx", - "add r8, 58368", - "call {vg_mldsa_bit_unpack}", + "add rsi, 73728", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 58368", + "add rdi, 33792", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 2816", - "mov esi, 416", - "mov edx, 4095", - "mov ecx, 4096", - "mov r8, rbx", - "add r8, 59392", - "call {vg_mldsa_bit_unpack}", + "add rsi, 74752", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 59392", + "add rdi, 33792", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 3232", - "mov esi, 416", - "mov edx, 4095", - "mov ecx, 4096", - "mov r8, rbx", - "add r8, 60416", - "call {vg_mldsa_bit_unpack}", + "add rsi, 75776", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 60416", + "add rdi, 33792", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 3648", - "mov esi, 416", - "mov edx, 4095", - "mov ecx, 4096", - "mov r8, rbx", - "add r8, 61440", - "call {vg_mldsa_bit_unpack}", + "add rsi, 76800", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 61440", + "add rdi, 33792", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 4064", - "mov esi, 416", - "mov edx, 4095", - "mov ecx, 4096", - "mov r8, rbx", - "add r8, 62464", - "call {vg_mldsa_bit_unpack}", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 62464", + "add rdi, 33792", "mov rsi, rbx", "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 4480", - "mov esi, 416", - "mov edx, 4095", - "mov ecx, 4096", - "mov r8, rbx", - "add r8, 63488", - "call {vg_mldsa_bit_unpack}", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 63488", + "add rdi, 34816", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbp", - "add rcx, 32", - "mov r8d, 32", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", + "add rsi, 78848", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 32", - "mov rcx, r13", - "add rcx, 0", - "mov r8d, 32", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 79872", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 64", - "mov rcx, r12", - "add rcx, 0", - "mov r8d, 64", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 80896", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 128", - "mov ecx, 31", - "mov r8, rbx", - "add r8, 200", - "call {vg_keccak_pad}", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 81920", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbx", - "add rcx, 960", - "mov r8d, 64", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_squeeze}", - "mov eax, 0", - "mov QWORD PTR [rbx+896], rax", - "mov eax, 814", - "mov QWORD PTR [rbx+888], rax", - "23:", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 0", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 82944", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 83968", "mov rdx, rbx", - "add rdx, 18432", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 34816", "mov rsi, rbx", - "add rsi, 18432", - "mov ecx, 1024", - "24:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 24b", + "add rsi, 84992", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 34816", "mov rsi, rbx", "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 1", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 86016", "mov rdx, rbx", - "add rdx, 19456", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 35840", "mov rsi, rbx", - "add rsi, 19456", - "mov ecx, 1024", - "25:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 25b", + "add rsi, 87040", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 35840", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 2", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", + "add rsi, 88064", "mov rdx, rbx", - "add rdx, 20480", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 27648", + "add rdi, 35840", "mov rsi, rbx", - "add rsi, 20480", - "mov ecx, 1024", - "26:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 26b", + "add rsi, 89088", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 27648", + "add rdi, 35840", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 3", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "add rsi, 90112", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 91136", "mov rdx, rbx", - "add rdx, 21504", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 28672", + "add rdi, 35840", "mov rsi, rbx", - "add rsi, 21504", - "mov ecx, 1024", - "27:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 27b", + "add rsi, 92160", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 28672", + "add rdi, 35840", "mov rsi, rbx", "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 4", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 93184", "mov rdx, rbx", - "add rdx, 22528", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 29696", + "add rdi, 36864", "mov rsi, rbx", - "add rsi, 22528", - "mov ecx, 1024", - "28:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 28b", + "add rsi, 94208", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 29696", + "add rdi, 36864", "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 5", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", + "add rsi, 95232", "mov rdx, rbx", - "add rdx, 23552", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 30720", - "mov rsi, rbx", - "add rsi, 23552", - "mov ecx, 1024", - "29:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 29b", - "mov rdi, rbx", - "add rdi, 30720", - "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 6", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 24576", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", - "mov rdi, rbx", - "add rdi, 31744", - "mov rsi, rbx", - "add rsi, 24576", - "mov ecx, 1024", - "210:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 210b", - "mov rdi, rbx", - "add rdi, 31744", - "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 32768", - "mov rsi, rbx", - "add rsi, 64512", - "mov rdx, rbx", - "add rdx, 25600", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 32768", - "mov rsi, rbx", - "add rsi, 65536", - "mov rdx, rbx", - "add rdx, 26624", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 32768", - "mov rsi, rbx", - "add rsi, 66560", - "mov rdx, rbx", - "add rdx, 27648", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 32768", + "add rdi, 36864", "mov rsi, rbx", - "add rsi, 67584", + "add rsi, 96256", "mov rdx, rbx", "add rdx, 28672", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 32768", + "add rdi, 36864", "mov rsi, rbx", - "add rsi, 68608", + "add rsi, 97280", "mov rdx, rbx", "add rdx, 29696", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 32768", + "add rdi, 36864", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 98304", "mov rdx, rbx", "add rdx, 30720", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 32768", + "add rdi, 36864", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 99328", "mov rdx, rbx", "add rdx, 31744", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 32768", + "add rdi, 36864", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 37888", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 100352", "mov rdx, rbx", "add rdx, 25600", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 37888", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 101376", "mov rdx, rbx", "add rdx, 26624", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 37888", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 102400", "mov rdx, rbx", "add rdx, 27648", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 37888", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 103424", "mov rdx, rbx", "add rdx, 28672", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 37888", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 104448", "mov rdx, rbx", "add rdx, 29696", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 37888", "mov rsi, rbx", - "add rsi, 76800", + "add rsi, 105472", "mov rdx, rbx", "add rdx, 30720", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 37888", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 106496", "mov rdx, rbx", "add rdx, 31744", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 37888", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 38912", "mov rsi, rbx", - "add rsi, 78848", + "add rsi, 107520", "mov rdx, rbx", "add rdx, 25600", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 38912", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 108544", "mov rdx, rbx", "add rdx, 26624", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 38912", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 109568", "mov rdx, rbx", "add rdx, 27648", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 38912", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 110592", "mov rdx, rbx", "add rdx, 28672", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 38912", "mov rsi, rbx", - "add rsi, 82944", + "add rsi, 111616", "mov rdx, rbx", "add rdx, 29696", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 38912", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 112640", "mov rdx, rbx", "add rdx, 30720", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 38912", "mov rsi, rbx", - "add rsi, 84992", + "add rsi, 113664", "mov rdx, rbx", "add rdx, 31744", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 38912", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 39936", "mov rsi, rbx", - "add rsi, 86016", + "add rsi, 114688", "mov rdx, rbx", "add rdx, 25600", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 39936", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 115712", "mov rdx, rbx", "add rdx, 26624", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 39936", "mov rsi, rbx", - "add rsi, 88064", + "add rsi, 116736", "mov rdx, rbx", "add rdx, 27648", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 39936", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 117760", "mov rdx, rbx", "add rdx, 28672", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 39936", "mov rsi, rbx", - "add rsi, 90112", + "add rsi, 118784", "mov rdx, rbx", "add rdx, 29696", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 39936", "mov rsi, rbx", - "add rsi, 91136", + "add rsi, 119808", "mov rdx, rbx", "add rdx, 30720", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 39936", "mov rsi, rbx", - "add rsi, 92160", + "add rsi, 120832", "mov rdx, rbx", "add rdx, 31744", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 39936", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 36864", - "mov rsi, rbx", - "add rsi, 93184", + "add rdi, 32768", + "mov esi, 261888", "mov rdx, rbx", - "add rdx, 25600", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", "mov rdi, rbx", - "add rdi, 36864", - "mov rsi, rbx", - "add rsi, 94208", + "add rdi, 6144", + "mov esi, 15", "mov rdx, rbx", - "add rdx, 26624", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 36864", - "mov rsi, rbx", - "add rsi, 95232", + "add rdi, 33792", + "mov esi, 261888", "mov rdx, rbx", - "add rdx, 27648", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", "mov rdi, rbx", - "add rdi, 36864", - "mov rsi, rbx", - "add rsi, 96256", + "add rdi, 6144", + "mov esi, 15", "mov rdx, rbx", - "add rdx, 28672", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2176", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 36864", - "mov rsi, rbx", - "add rsi, 97280", + "add rdi, 34816", + "mov esi, 261888", "mov rdx, rbx", - "add rdx, 29696", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", "mov rdi, rbx", - "add rdi, 36864", - "mov rsi, rbx", - "add rsi, 98304", + "add rdi, 6144", + "mov esi, 15", "mov rdx, rbx", - "add rdx, 30720", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2304", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 36864", - "mov rsi, rbx", - "add rsi, 99328", + "add rdi, 35840", + "mov esi, 261888", "mov rdx, rbx", - "add rdx, 31744", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2432", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 36864", - "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_inv_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 37888", - "mov rsi, rbx", - "add rsi, 100352", - "mov rdx, rbx", - "add rdx, 25600", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 37888", - "mov rsi, rbx", - "add rsi, 101376", - "mov rdx, rbx", - "add rdx, 26624", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 37888", - "mov rsi, rbx", - "add rsi, 102400", - "mov rdx, rbx", - "add rdx, 27648", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 37888", - "mov rsi, rbx", - "add rsi, 103424", - "mov rdx, rbx", - "add rdx, 28672", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 37888", - "mov rsi, rbx", - "add rsi, 104448", - "mov rdx, rbx", - "add rdx, 29696", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 37888", - "mov rsi, rbx", - "add rsi, 105472", - "mov rdx, rbx", - "add rdx, 30720", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 37888", - "mov rsi, rbx", - "add rsi, 106496", - "mov rdx, rbx", - "add rdx, 31744", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 37888", - "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_inv_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 38912", - "mov rsi, rbx", - "add rsi, 107520", - "mov rdx, rbx", - "add rdx, 25600", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 38912", - "mov rsi, rbx", - "add rsi, 108544", - "mov rdx, rbx", - "add rdx, 26624", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 38912", - "mov rsi, rbx", - "add rsi, 109568", - "mov rdx, rbx", - "add rdx, 27648", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 38912", - "mov rsi, rbx", - "add rsi, 110592", - "mov rdx, rbx", - "add rdx, 28672", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 38912", - "mov rsi, rbx", - "add rsi, 111616", - "mov rdx, rbx", - "add rdx, 29696", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 38912", - "mov rsi, rbx", - "add rsi, 112640", - "mov rdx, rbx", - "add rdx, 30720", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 38912", - "mov rsi, rbx", - "add rsi, 113664", - "mov rdx, rbx", - "add rdx, 31744", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 38912", - "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_inv_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 39936", - "mov rsi, rbx", - "add rsi, 114688", - "mov rdx, rbx", - "add rdx, 25600", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 39936", - "mov rsi, rbx", - "add rsi, 115712", - "mov rdx, rbx", - "add rdx, 26624", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 39936", - "mov rsi, rbx", - "add rsi, 116736", - "mov rdx, rbx", - "add rdx, 27648", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 39936", - "mov rsi, rbx", - "add rsi, 117760", - "mov rdx, rbx", - "add rdx, 28672", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 39936", - "mov rsi, rbx", - "add rsi, 118784", - "mov rdx, rbx", - "add rdx, 29696", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 39936", - "mov rsi, rbx", - "add rsi, 119808", - "mov rdx, rbx", - "add rdx, 30720", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 39936", - "mov rsi, rbx", - "add rsi, 120832", - "mov rdx, rbx", - "add rdx, 31744", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 39936", - "mov rsi, rbx", - "add rsi, 3072", - "call {vg_mldsa_inv_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 32768", - "mov esi, 261888", - "mov rdx, rbx", - "add rdx, 6144", - "call {vg_mldsa_high_bits}", - "mov rdi, rbx", - "add rdi, 6144", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 2048", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 33792", - "mov esi, 261888", - "mov rdx, rbx", - "add rdx, 6144", - "call {vg_mldsa_high_bits}", - "mov rdi, rbx", - "add rdi, 6144", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 2176", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 34816", - "mov esi, 261888", - "mov rdx, rbx", - "add rdx, 6144", - "call {vg_mldsa_high_bits}", - "mov rdi, rbx", - "add rdi, 6144", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 2304", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 35840", - "mov esi, 261888", - "mov rdx, rbx", - "add rdx, 6144", - "call {vg_mldsa_high_bits}", - "mov rdi, rbx", - "add rdi, 6144", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 2432", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 36864", - "mov esi, 261888", - "mov rdx, rbx", - "add rdx, 6144", - "call {vg_mldsa_high_bits}", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", "mov rdi, rbx", "add rdi, 6144", "mov esi, 15", @@ -7341,1400 +6549,608 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 0", "mov esi, 136", "mov edx, 32", - "mov rcx, rbx", - "add rcx, 896", - "mov r8d, 2", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 34", - "mov ecx, 31", - "mov r8, rbx", - "add r8, 200", - "call {vg_keccak_pad}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbx", - "add rcx, 1024", - "mov r8d, 128", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_squeeze}", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "20:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 20b", - "mov rdi, rbx", - "add rdi, 1216", - "mov rsi, rbx", - "add rsi, 1056", - "mov ecx, 64", - "21:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 21b", - "mov eax, 0", - "mov BYTE PTR [rbx+1281], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 4096", - "mov ecx, 256", - "22:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 22b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 5120", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 5120", - "mov ecx, 256", - "23:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 23b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 6144", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 6144", - "mov ecx, 256", - "24:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 24b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 7168", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 7168", - "mov ecx, 256", - "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 25b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 8192", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 8192", - "mov ecx, 256", - "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 26b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 9216", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 9216", - "mov ecx, 256", - "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 27b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 10240", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 10240", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 11264", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 11264", - "mov ecx, 256", - "29:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 29b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 12288", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 12288", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 13312", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 13312", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 14336", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 14336", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 15360", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 15360", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 16384", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 16384", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 17408", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 17408", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 18432", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 18432", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 19456", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 19456", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 20480", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 20480", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 21504", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 21504", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 22528", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 22528", - "mov ecx, 256", - "220:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 220b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 23552", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 23552", - "mov ecx, 256", - "221:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 221b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 24576", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 24576", - "mov ecx, 256", - "222:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 222b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 25600", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 25600", - "mov ecx, 256", - "223:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 223b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 26624", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 26624", - "mov ecx, 256", - "224:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 224b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 27648", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 27648", - "mov ecx, 256", - "225:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 225b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 33792", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 33792", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 34816", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 34816", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 35840", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 35840", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 36864", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "236:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 236b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "mov rcx, rbx", + "add rcx, 896", + "mov r8d, 2", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 39936", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 0", + "mov esi, 136", + "mov edx, 34", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "237:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 237b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 128", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 40960", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", - "238:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 238b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 20b", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1216", "mov rsi, rbx", - "add rsi, 41984", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 41984", - "mov ecx, 256", - "239:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1056", + "mov ecx, 64", + "21:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 239b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 21b", + "mov eax, 0", + "mov BYTE PTR [rbx+1281], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 43008", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 43008", - "mov ecx, 256", - "240:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "22:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 240b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 22b", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1442", "mov rsi, rbx", - "add rsi, 44032", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 44032", - "mov ecx, 256", - "241:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "23:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 241b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 23b", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1476", "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "242:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 242b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 24b", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1510", "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "243:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "25:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 243b", + "jne 25b", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 4096", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "244:", + "add rdi, 4096", + "mov ecx, 1024", + "26:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 244b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "jne 26b", + "mov eax, 4", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1475], al", "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "245:", + "add rdi, 8192", + "mov ecx, 1024", + "27:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 245b", + "jne 27b", + "mov eax, 1", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1441], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", - "246:", + "add rdi, 12288", + "mov ecx, 1024", + "28:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 246b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "jne 28b", + "mov eax, 5", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1441], al", "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 16384", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 50176", - "mov ecx, 256", - "247:", + "add rdi, 16384", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 247b", + "jne 29b", + "mov eax, 2", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1475], al", "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 51200", + "add rsi, 20480", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 51200", - "mov ecx, 256", - "248:", + "add rdi, 20480", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 248b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", + "jne 210b", "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 52224", + "add rsi, 24576", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 52224", - "mov ecx, 256", - "249:", + "add rdi, 24576", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 249b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", + "jne 211b", + "mov eax, 3", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1509], al", "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "250:", + "add rdi, 28672", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 250b", + "jne 212b", "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 32768", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "251:", + "add rdi, 32768", + "mov ecx, 1024", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 251b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "jne 213b", + "mov eax, 4", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "252:", + "add rdi, 36864", + "mov ecx, 1024", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 252b", + "jne 214b", + "mov eax, 1", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1441], al", "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 40960", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "253:", + "add rdi, 40960", + "mov ecx, 1024", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 253b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "jne 215b", + "mov eax, 5", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "254:", + "add rdi, 45056", + "mov ecx, 1024", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 254b", + "jne 216b", + "mov eax, 2", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1475], al", "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 49152", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "255:", + "add rdi, 49152", + "mov ecx, 1024", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 255b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", + "jne 217b", + "mov eax, 6", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1474], al", "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1509], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1542], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "256:", + "add rdi, 53248", + "mov ecx, 1024", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 256b", + "jne 218b", + "mov eax, 3", + "mov BYTE PTR [rbx+1440], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1441], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1474], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1475], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1508], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1509], al", "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", + "mov BYTE PTR [rbx+1542], al", "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1543], al", "mov rdi, rbx", - "add rdi, 1152", + "add rdi, 1408", "mov rsi, rbx", - "add rsi, 60416", + "add rsi, 57344", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 79872", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 60416", - "mov ecx, 256", - "257:", + "add rdi, 57344", + "mov ecx, 1024", + "219:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 257b", + "jne 219b", "mov eax, 0", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8751,13 +7167,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 61440", "mov ecx, 256", - "258:", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 258b", + "jne 220b", "mov eax, 1", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8774,13 +7190,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 62464", "mov ecx, 256", - "259:", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 259b", + "jne 221b", "mov eax, 2", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8797,13 +7213,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 63488", "mov ecx, 256", - "260:", + "222:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 260b", + "jne 222b", "mov eax, 3", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8820,13 +7236,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 64512", "mov ecx, 256", - "261:", + "223:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 261b", + "jne 223b", "mov eax, 4", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8843,13 +7259,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 65536", "mov ecx, 256", - "262:", + "224:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 262b", + "jne 224b", "mov eax, 5", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8866,13 +7282,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 66560", "mov ecx, 256", - "263:", + "225:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 263b", + "jne 225b", "mov eax, 6", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8889,13 +7305,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 67584", "mov ecx, 256", - "264:", + "226:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 264b", + "jne 226b", "mov eax, 7", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8912,13 +7328,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 68608", "mov ecx, 256", - "265:", + "227:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 265b", + "jne 227b", "mov eax, 8", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8935,13 +7351,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 69632", "mov ecx, 256", - "266:", + "228:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 266b", + "jne 228b", "mov eax, 9", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8958,13 +7374,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 70656", "mov ecx, 256", - "267:", + "229:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 267b", + "jne 229b", "mov eax, 10", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -8981,13 +7397,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 71680", "mov ecx, 256", - "268:", + "230:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 268b", + "jne 230b", "mov eax, 11", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -9004,13 +7420,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 72704", "mov ecx, 256", - "269:", + "231:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 269b", + "jne 231b", "mov eax, 12", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -9027,13 +7443,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 73728", "mov ecx, 256", - "270:", + "232:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 270b", + "jne 232b", "mov eax, 13", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -9050,13 +7466,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 74752", "mov ecx, 256", - "271:", + "233:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 271b", + "jne 233b", "mov eax, 14", "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", @@ -9073,49 +7489,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 75776", "mov ecx, 256", - "272:", + "234:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 272b", + "jne 234b", "mov rdi, r12", "add rdi, 0", "mov rsi, rbx", "add rsi, 1024", "mov ecx, 32", - "273:", + "235:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 273b", + "jne 235b", "mov rdi, r13", "add rdi, 0", "mov rsi, rbx", "add rsi, 1024", "mov ecx, 32", - "274:", + "236:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 274b", + "jne 236b", "mov rdi, r13", "add rdi, 32", "mov rsi, rbx", "add rsi, 1120", "mov ecx, 32", - "275:", + "237:", "movzx eax, BYTE PTR [rsi]", "mov BYTE PTR [rdi], al", "add rdi, 1", "add rsi, 1", "sub rcx, 1", - "jne 275b", + "jne 237b", "mov rdi, rbx", "add rdi, 61440", "mov esi, 2", @@ -9984,7 +8400,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt,