diff --git a/README.md b/README.md
index 4ffdc5109..8e0fc01f1 100644
--- a/README.md
+++ b/README.md
@@ -891,7 +891,7 @@ yours to keep:
✅ |
-✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) |
+✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2) |
✅ SHA extensions |
@@ -907,7 +907,7 @@ yours to keep:
✅ |
-✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) |
+✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2) |
✅ SHA extensions |
@@ -923,7 +923,7 @@ yours to keep:
✅ |
-✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) |
+✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2) |
✅ SHA extensions |
diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml
index 4f28100c0..23c04ea14 100644
--- a/docs/algorithms/ml-dsa-44.toml
+++ b/docs/algorithms/ml-dsa-44.toml
@@ -3,4 +3,4 @@ family = "Signatures"
specs = ["MlDsa"]
modules = ["src/mldsa44.rs"]
asm = ["mldsa44", "mldsa"]
-optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" }
+optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2)" }
diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml
index 6683e2898..04ce839cb 100644
--- a/docs/algorithms/ml-dsa-65.toml
+++ b/docs/algorithms/ml-dsa-65.toml
@@ -3,4 +3,4 @@ family = "Signatures"
specs = ["MlDsa"]
modules = ["src/mldsa65.rs"]
asm = ["mldsa65", "mldsa"]
-optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" }
+optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2)" }
diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml
index 649298bea..88e39d47a 100644
--- a/docs/algorithms/ml-dsa-87.toml
+++ b/docs/algorithms/ml-dsa-87.toml
@@ -3,4 +3,4 @@ family = "Signatures"
specs = ["MlDsa"]
modules = ["src/mldsa87.rs"]
asm = ["mldsa87", "mldsa"]
-optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" }
+optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in key generation and verification (four SHAKE128 instances at once with AVX2)" }
diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean
index 5206623ad..56aac4f8e 100644
--- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean
+++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean
@@ -29,6 +29,7 @@ def prims : Prims where
power2Round := Round.power2Round
simpleBitPack := Pack.simpleBitPack
bitPack := Pack.bitPack
+ rej4 := Sample.Rej4.rejNTT4
/-- The primitives, with the polynomial arithmetic of `B`. -/
def primsWith (B : Arith.Backend) : Prims :=
@@ -38,6 +39,7 @@ def primsWith (B : Arith.Backend) : Prims :=
mul := B.mul
mulAdd := B.mulAdd
add := B.add
+ rej4 := B.rej4
sfx := B.sfx }
end VG.Impl.MlDsa.X86_64.KeyGen
diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean
index 4f3d0ef4c..8335353dd 100644
--- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean
+++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean
@@ -17,18 +17,22 @@ The layout of `scratch` (in bytes): the Keccak state at 0 and the sponge
functions' working space at 200, the saved registers at 840 (as ML-KEM's);
`k` and `ℓ` at 896 (`oKL`); `(ρ, ρ′, K)` at 1024 (`oHX`, 128 bytes); the
seed of `RejNTTPoly` at 1152 (`oSA`, 34 bytes) and of `RejBoundedPoly` at
-1216 (`oSB`, 66 bytes); the working space of the primitives at 2048
+1216 (`oSB`, 66 bytes); the four seeds of `vg_mldsa_rej_ntt_poly4` at 1408
+(`oSA4`, 136 bytes); the working space of the primitives at 2048
(2048 bytes); and polynomials of 1024 bytes from 4096 (`oP j`): `Â[r, s]`
is polynomial `rℓ + s`, `s₁[j]` (then `ŝ₁[j]`) polynomial `kℓ + j`,
`s₂[i]` polynomial `kℓ + ℓ + i`, and `t`, `t₁` and `t₀` the three after
-them.
+them; then the working space of `vg_mldsa_rej_ntt_poly4` (8 KiB, `oR4`).
1. `(ρ, ρ′, K) = H(ξ ‖ k ‖ ℓ, 128)`, and `ρ` and `ρ′` to the seeds.
-2. `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)` and `s₁ ‖ s₂ = RejBoundedPoly(ρ′ ‖ r ‖ 0)`
- (`ExpandA`, `ExpandS`). After each, `r15 ← r15 ∧ result`, and the
- polynomial is ANDed with `-result` (`mask`): it is zero if the sampler
- failed, so that every polynomial is reduced, and small, whatever the
- samplers return, without a branch.
+2. `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)`, four consecutive entries at a time
+ (`vg_mldsa_rej_ntt_poly4`, from the seeds at `oSA4`, each `ρ` with its
+ entry's indices) and the last `kℓ mod 4` one at a time, and
+ `s₁ ‖ s₂ = RejBoundedPoly(ρ′ ‖ r ‖ 0)` (`ExpandA`, `ExpandS`). After each
+ call, `r15 ← r15 ∧ result`, and the polynomials it sampled are ANDed with
+ `-result` (`mask`): they are zero if the sampler failed, so that every
+ polynomial is reduced, and small, whatever the samplers return, without a
+ branch.
3. `ρ` and `K` to `sk`; `s₁` and `s₂`, `BitPack`ed, to `sk`; `ŝ₁ = NTT(s₁)`.
4. For each row `i`: `t = NTT⁻¹(Σⱼ Â[i, j] ŝ₁[j]) + s₂[i]`, `Power2Round`, and
`t₁` `SimpleBitPack`ed to `pk`, `t₀` `BitPack`ed to `sk`; `ρ` to `pk`.
@@ -50,6 +54,7 @@ def oKL : Nat := 896
def oHX : Nat := 1024
def oSA : Nat := 1152
def oSB : Nat := 1216
+def oSA4 : Nat := 1408
/-- Polynomial `j`. -/
def oP (j : Nat) : Nat := 4096 + 1024 * j
@@ -60,6 +65,8 @@ abbrev sP (p : Params) (r : Nat) : Ptr := sc (oP (p.k * p.ℓ + r))
abbrev tP (p : Params) : Ptr := sc (oP (p.k * p.ℓ + p.ℓ + p.k))
abbrev t1P (p : Params) : Ptr := sc (oP (p.k * p.ℓ + p.ℓ + p.k + 1))
abbrev t0P (p : Params) : Ptr := sc (oP (p.k * p.ℓ + p.ℓ + p.k + 2))
+/-- The working space of `vg_mldsa_rej_ntt_poly4`. -/
+def oR4 (p : Params) : Nat := oP (p.k * p.ℓ + p.ℓ + p.k + 3)
/-- The length of a packed polynomial of `s₁` or `s₂`, `32 · bitlen (2η)`. -/
def lenS (p : Params) : Nat := 32 * bitlen (2 * p.η)
@@ -89,6 +96,9 @@ def addAt (sfx : String) (c : Prog isa) (f g : Ptr) : Prog isa :=
def rejNttAt (c : Prog isa) (seed a : Ptr) : Prog isa :=
.seq (.block (lea .rdi seed ++ lea .rsi a ++ lea .rdx (sc oSS))) (.call "vg_mldsa_rej_ntt_poly" c)
+def rej4At (c : Prog isa) (sfx : String) (a w : Ptr) : Prog isa :=
+ .seq (.block (lea .rdi (sc oSA4) ++ lea .rsi a ++ lea .rdx w)) (.call ("vg_mldsa_rej_ntt_poly4" ++ sfx) c)
+
def rejBoundedAt (c : Prog isa) (seed : Ptr) (eta : Nat) (a : Ptr) : Prog isa :=
.seq (.block (lea .rdi seed ++ imm .rsi eta ++ lea .rdx a ++ lea .rcx (sc oSS)))
(.call "vg_mldsa_rej_bounded_poly" c)
@@ -104,10 +114,11 @@ def bitPackAt (c : Prog isa) (f : Ptr) (a b : Nat) (out : Ptr) (len : Nat) : Pro
.seq (.block (lea .rdi f ++ imm .rsi a ++ imm .rdx b ++ lea .rcx out ++ imm .r8 len))
(.call "vg_mldsa_bit_pack" c)
-/-- `r15 ← r15 ∧ eax`, and the polynomial at `a` ANDed with `-eax` (`eax` is 0 or 1). -/
-def mask (a : Ptr) : Prog isa :=
+/-- `r15 ← r15 ∧ eax`, and the polynomial at `a` (or the `N` coefficients from `a`) ANDed with `-eax`
+(`eax` is 0 or 1). -/
+def mask (a : Ptr) (N : Nat := 256) : Prog isa :=
.seq (.block ([.alu32 .and .r15 (.reg .rax), .mov32 .r8 (.imm 0), .alu32 .sub .r8 (.reg .rax)] ++
- lea .rdi a ++ imm .rcx 256))
+ lea .rdi a ++ imm .rcx N))
(.loop (.block [.mov32 .rax (.mem (at_ .rdi 0)), .alu32 .and .rax (.reg .r8), .store32 (at_ .rdi 0) .rax,
.alu .add .rdi (.imm 4), .alu .sub .rcx (.imm 1)]) .ne)
@@ -120,13 +131,28 @@ def pro : List Instr := topPro .rcx [(.rbp, .rdi), (.r12, .rsi), (.r13, .rdx)]
def seeds (p : Params) : Prog isa :=
.seq (.block (setB (sc oKL) p.k ++ setB (sc (oKL + 1)) p.ℓ))
(.seq (hashAt [((.rbp, 0), 32), (sc oKL, 2)] 136 0x1f (sc oHX) 128)
- (.seq (copy (sc oSA) (sc oHX) 32) (.seq (copy (sc oSB) (sc (oHX + 32)) 64) (.block (setB (sc (oSB + 65)) 0)))))
+ (.seq (copy (sc oSA) (sc oHX) 32) (.seq (copy (sc oSB) (sc (oHX + 32)) 64) (.seq (.block (setB (sc (oSB + 65)) 0))
+ (.seq (copy (sc oSA4) (sc oHX) 32) (.seq (copy (sc (oSA4 + 34)) (sc oHX) 32)
+ (.seq (copy (sc (oSA4 + 68)) (sc oHX) 32) (copy (sc (oSA4 + 102)) (sc oHX) 32))))))))
/-- `Â[e / ℓ, e % ℓ] = RejNTTPoly(ρ ‖ e % ℓ ‖ e / ℓ)`. -/
def expA (P : Prims) (p : Params) (e : Nat) : Prog isa :=
.seq (.block (setB (sc (oSA + 32)) (e % p.ℓ) ++ setB (sc (oSA + 33)) (e / p.ℓ)))
(.seq (rejNttAt P.rejNtt (sc oSA) (aP e)) (mask (aP e)))
+/-- The indices of entry `e + k` of `Â` to seed `k` of `oSA4`. -/
+def setSR (p : Params) (e k : Nat) : List Instr :=
+ setB (sc (oSA4 + 34 * k + 32)) ((e + k) % p.ℓ) ++ setB (sc (oSA4 + 34 * k + 32 + 1)) ((e + k) / p.ℓ)
+
+/-- Entries `4g, …, 4g + 3` of `Â`. -/
+def expA4 (P : Prims) (p : Params) (g : Nat) : Prog isa :=
+ .seq (.block (setSR p (4 * g) 0)) (.seq (.block (setSR p (4 * g) 1)) (.seq (.block (setSR p (4 * g) 2))
+ (.seq (.block (setSR p (4 * g) 3)) (.seq (rej4At P.rej4 P.sfx (aP (4 * g)) (sc (oR4 p))) (mask (aP (4 * g)) 1024)))))
+
+/-- The entries of `Â`: four at a time, then the last `kℓ mod 4` one at a time. -/
+def expAll (P : Prims) (p : Params) : Prog isa :=
+ .seq (seqR (expA4 P p) 0 (p.k * p.ℓ / 4)) (seqR (expA P p) (4 * (p.k * p.ℓ / 4)) (p.k * p.ℓ % 4))
+
/-- Entry `r` of `s₁ ‖ s₂`: `RejBoundedPoly(ρ′ ‖ r ‖ 0)`. -/
def expS (P : Prims) (p : Params) (r : Nat) : Prog isa :=
.seq (.block (setB (sc (oSB + 64)) r))
@@ -162,7 +188,7 @@ def rest (P : Prims) (p : Params) : Prog isa :=
/-- `vg_mldsa*_keygen` for the parameter set `p`, calling the primitives `P`. -/
def keyGen (P : Prims) (p : Params) : Prog isa :=
- .seq (.block pro) (.seq (seeds p) (.seq (seqR (expA P p) 0 (p.k * p.ℓ))
+ .seq (.block pro) (.seq (seeds p) (.seq (expAll P p)
(.seq (seqR (expS P p) 0 (p.ℓ + p.k)) (.seq (rest P p) (.block topEpi)))))
end VG.Impl.MlDsa.X86_64.KeyGen
diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean
index a2ed604be..9c33d6845 100644
--- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean
+++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean
@@ -36,6 +36,8 @@ structure Prims where
simpleBitPack : Prog isa
/-- `vg_mldsa_bit_pack` -/
bitPack : Prog isa
+ /-- `vg_mldsa_rej_ntt_poly4` -/
+ rej4 : Prog isa
/-- What the names of the polynomial arithmetic's functions end with (`Arith.Backend`). -/
sfx : String := ""
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Base.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Base.lean
index a8ab5a6ef..54a459c66 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Base.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Base.lean
@@ -32,6 +32,13 @@ structure Callee (c : Prog isa) (k : Nat → Contract isa) : Prop where
nosp : NoSp c
depth : c.depth ≤ 2
+/-- `vg_mldsa_rej_ntt_poly4`: verified with 24 bytes of stack, never writing `rsp` but by calls nested at
+most three deep. -/
+structure Callee4 (c : Prog isa) : Prop where
+ verified : Verified X86_64.target c (Spec.MlDsa.rejNTT4Contract X86_64.abi 24)
+ nosp : NoSp c
+ depth : c.depth ≤ 3
+
/-- Verified implementations of the primitives key generation calls. -/
structure PrimsOk (P : Prims) : Prop where
ntt : Callee P.ntt (fun stk => Spec.MlDsa.nttContract X86_64.abi stk)
@@ -44,6 +51,7 @@ structure PrimsOk (P : Prims) : Prop where
power2Round : Callee P.power2Round (fun stk => Spec.MlDsa.power2RoundContract X86_64.abi stk)
simpleBitPack : Callee P.simpleBitPack (fun stk => Spec.MlDsa.simpleBitPackContract X86_64.abi stk)
bitPack : Callee P.bitPack (fun stk => Spec.MlDsa.bitPackContract X86_64.abi stk)
+ rej4 : Callee4 P.rej4
/-! ## MXCSR -/
@@ -118,7 +126,7 @@ theorem WP.callMx {n : String} {c : Prog isa} {k : Contract isa}
/-- `glueCall_ok`, keeping MXCSR's control bits. -/
theorem glueCallMx_ok {glue : List Instr} {n : String} {c : Prog isa} {k : Contract isa}
(hv : ∀ s, k.pre s → ∃ t s', Exec isa c s t s' ∧ abiPreserved s s' ∧ k.post s s')
- (hsp : NoSp c) (hd : c.depth ≤ 2) (hgl : ∀ i ∈ glue, loadsMxcsr i = false) {s : State}
+ (hsp : NoSp c) (hd : c.depth ≤ 3) (hgl : ∀ i ∈ glue, loadsMxcsr i = false) {s : State}
{V : State → Prop}
(hg : WP isa (.block glue) s fun s1 => (V s1 ∧ s1.mem = s.mem) ∧ Keep MlKem.X86_64.argRegs s s1)
{rd wr : List Region} (hpre : ∀ s1, V s1 → s1.mem = s.mem → Keep MlKem.X86_64.argRegs s s1 →
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean
index f8a03397c..8b7a2eed6 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean
@@ -44,6 +44,19 @@ theorem ceWf (h24 : 24 ≤ (s.gpr .rsp).toNat) {n : Nat} (hn : n + 1 ≤ 16) : n
have := (s.gpr .rsp).isLt
omega
+omit L in
+theorem ceWf24 (h32 : 32 ≤ (s.gpr .rsp).toNat) : 24 ≤ (s1.gpr .rsp - 8).toNat := by
+ rw [hsp, BitVec.toNat_sub]
+ have : (8 : BitVec 64).toNat = 8 := rfl
+ rw [this]
+ have := (s.gpr .rsp).isLt
+ omega
+
+theorem ceD24 {q : Ptr} {l : Nat} (h : inB (kgB p) q l = true) :
+ Region.Disjoint (below (s1.gpr .rsp - 8) 24) ⟨pa s q, l⟩ := by
+ have := stk_disj24 s1 (R := ⟨pa s q, l⟩) (by rw [hsp]; exact L.stkD h)
+ simpa only [State.callEntry_rsp] using this
+
end
/-! ## The memory of a call's entry -/
@@ -93,7 +106,7 @@ theorem primOk {c : Prog isa} {kk : Nat → Contract isa} (hc : Callee c kk) {gl
∃ s₂ : State, s₂.mem = s'.mem ∧ (∀ r, r ≠ .rsp → s₂.gpr r = s'.gpr r) ∧
(kk stk).post (s1.callEntry.withRegions rd wr) s₂ := by
obtain ⟨stk, hs, hver⟩ := hc.verified
- exact WP.mono (glueCallMx_ok hver.1 hc.nosp hc.depth hgl hg (hpre stk hs) hcov hw)
+ exact WP.mono (glueCallMx_ok hver.1 hc.nosp (Nat.le_succ_of_le hc.depth) hgl hg (hpre stk hs) hcov hw)
fun s' ⟨h1, h2, h3⟩ => ⟨h1, h2, stk, hs, h3⟩
/-- A block of moves leaks nothing. -/
@@ -188,7 +201,9 @@ theorem covers2 {s : State} {p : Params} (L : Lay kgR (kgW p) s) {a b : Ptr} {la
/-- A state of the function, where a call can be made. -/
structure Site (p : Params) (s : State) : Prop where
lay : Lay kgR (kgW p) s
- h24 : 24 ≤ (s.gpr .rsp).toNat
+ h32 : 32 ≤ (s.gpr .rsp).toNat
+
+theorem Site.h24 {p : Params} {s : State} (S : Site p s) : 24 ≤ (s.gpr .rsp).toNat := by have := S.h32; omega
/-- The registers that hold the pointers of the layout. -/
abbrev kgRegs : List Reg := [.rbx, .rbp, .r12, .r13]
@@ -594,6 +609,86 @@ theorem rejNttAt_tr {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.rejNTTCo
end
+/-! ## `RejNTTPoly` four times -/
+
+section
+variable {p : Params} {a w : Ptr} (ha : PtrOk a) (hw : PtrOk w)
+ (h1 : sepB (kgB p) (sc oSA4) 136 a 4096 = true) (h2 : sepB (kgB p) (sc oSA4) 136 w 8192 = true)
+ (h3 : sepB (kgB p) a 4096 w 8192 = true)
+ (w1 : inB (kgW p) a 4096 = true) (w2 : inB (kgW p) w 8192 = true)
+
+include h1 h2 h3 in
+theorem rej4_pre {s s1 : State} (S : Site p s)
+ (hv : s1.gpr .rdi = pa s (sc oSA4) ∧ s1.gpr .rsi = pa s a ∧ s1.gpr .rdx = pa s w)
+ (k : Keep MlKem.X86_64.argRegs s s1) :
+ (Spec.MlDsa.rejNTT4Contract X86_64.abi 24).pre
+ (s1.callEntry.withRegions [⟨pa s (sc oSA4), 136⟩] [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩]) := by
+ obtain ⟨i1, i2, _⟩ := sepB_spec h1
+ obtain ⟨_, i3, _⟩ := sepB_spec h3
+ have L := S.lay
+ have hsp := keep_rsp k
+ sig_pre [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs]
+ simp only [hv.1, hv.2.1, hv.2.2]
+ exact ⟨ceWf24 hsp S.h32, trivial, trivial, L.disj h1, L.disj h2, L.disj h3, ceD1 L hsp i1, ceD1 L hsp i2,
+ ceD1 L hsp i3, ceD24 L hsp i1, ceD24 L hsp i2, ceD24 L hsp i3, L.nwp i1, L.nwp i2, L.nwp i3⟩
+
+include ha hw h1 h2 h3 w1 w2 in
+theorem rej4At_ok {c : Prog isa} {sfx : String} (hc : Callee4 c) {s : State} (S : Site p s) :
+ WP isa (rej4At c sfx a w) s fun s' => Post s s' [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩] ∧ MX s' = MX s ∧
+ ((s'.gpr .rax).setWidth 32 = 1 → ∀ k < 4, Spec.MlDsa.Reduced s'.mem (Spec.MlDsa.poly4 (pa s a) k)) ∧
+ (((s'.gpr .rax).setWidth 32 = 1 ∧ ∀ k < 4, ∃ b : Spec.MlDsa.Bounds,
+ Spec.MlDsa.rejNTTPoly b.rejNTT (Spec.MlDsa.seed4 s.mem (pa s (sc oSA4)) k) =
+ some (Spec.MlDsa.polyAt s'.mem (Spec.MlDsa.poly4 (pa s a) k))) ∨
+ ((s'.gpr .rax).setWidth 32 = 0 ∧ ∃ k < 4,
+ Spec.MlDsa.rejNTTPoly Spec.MlDsa.minBounds.rejNTT (Spec.MlDsa.seed4 s.mem (pa s (sc oSA4)) k) = none)) := by
+ obtain ⟨i1, _, _⟩ := sepB_spec h1
+ have L := S.lay
+ refine WP.mono (glueCallMx_ok hc.verified.1 hc.nosp hc.depth
+ (noLd_append (noLd_append (lea_noLd _ _) (lea_noLd _ _)) (lea_noLd _ _))
+ (glue3_ok (sc_ok oSA4 (by decide)) ha hw s) (fun s1 hv _ k => rej4_pre h1 h2 h3 S hv k)
+ (covers_rww L i1 w1 w2) (covers2 L w1 w2))
+ fun s' ⟨hP, hx, s1, hv, hm, k, s₂, hm₂, hg₂, hpost⟩ => ⟨hP, hx, ?_⟩
+ have hsp := keep_rsp k
+ sig_post [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] at hpost
+ simp only [hv.1, hv.2.1, hm₂, hg₂ .rax (by decide)] at hpost
+ have hseed : ∀ k < 4, Spec.MlDsa.seed4 (ceM s1) (pa s (sc oSA4)) k = Spec.MlDsa.seed4 s.mem (pa s (sc oSA4)) k :=
+ fun k hk => by
+ unfold Spec.MlDsa.seed4
+ rw [← hm]
+ refine Proof.MlKem.bytesAt_congr fun i hi => ?_
+ rw [BitVec.add_assoc, ← BitVec.ofNat_add]
+ exact callEntry_bytes s1 (R := ⟨pa s (sc oSA4), 136⟩) (k16 s1 (by rw [hsp]; exact L.stkD i1))
+ (show 136 ≤ 2 ^ 64 by decide) (show 34 * k + i < 136 by omega)
+ obtain ⟨hr, ho⟩ := hpost
+ refine ⟨hr, ?_⟩
+ rcases ho with ⟨h1', hb⟩ | ⟨h0, k, hk, hn⟩
+ · exact .inl ⟨h1', fun k hk => by rw [← hseed k hk]; exact hb k hk⟩
+ · exact .inr ⟨h0, k, hk, by rw [← hseed k hk]; exact hn⟩
+
+include ha hw h1 h2 h3 w1 w2 in
+theorem rej4At_tr {c : Prog isa} {sfx : String} (hc : Callee4 c) (hba : a.1 ∈ kgRegs) (hbw : w.1 ∈ kgRegs) :
+ RelCT isa (fun x y => Two p x y ∧ bytesAt x.mem (pa x (sc oSA4)) 136 = bytesAt y.mem (pa y (sc oSA4)) 136)
+ (rej4At c sfx a w) fun _ _ => True := by
+ obtain ⟨i1, _, _⟩ := sepB_spec h1
+ have g := fun x => glue3_ok (sc_ok oSA4 (by decide)) ha hw x
+ refine glueCall_tr hc.verified.1 hc.verified.2.1
+ (moves_tr (nomem_append (nomem_append (lea_nomem _ _) (lea_nomem _ _)) (lea_nomem _ _)))
+ (fun x y _ => ⟨g x, g y⟩)
+ fun x y x1 y1 ⟨T, e⟩ ⟨⟨hv1, hm1⟩, k1⟩ ⟨⟨hv2, hm2⟩, k2⟩ =>
+ ⟨_, _, _, _, rej4_pre h1 h2 h3 T.sx hv1 k1, rej4_pre h1 h2 h3 T.sy hv2 k2, ?_,
+ by rw [k1.2.1, k1.2.2]; exact covers_rww T.sx.lay i1 w1 w2, by rw [k1.2.2]; exact covers2 T.sx.lay w1 w2,
+ by rw [k2.2.1, k2.2.2]; exact covers_rww T.sy.lay i1 w1 w2, by rw [k2.2.2]; exact covers2 T.sy.lay w1 w2,
+ by rw [keep_rsp k1, keep_rsp k2, T.rsp]⟩
+ sig_pub [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs]
+ simp only [hv1.1, hv1.2.1, hv1.2.2, hv2.1, hv2.2.1, hv2.2.2, T.pa hba, T.pa hbw,
+ T.pa (q := sc oSA4) (by decide), and_true]
+ refine ⟨by rw [keep_rsp k1, keep_rsp k2, T.rsp], ?_⟩
+ rw [T.pa (q := sc oSA4) (by decide)] at e
+ rw [ce_bytesAt' (by decide) (by rw [keep_rsp k1, ← T.pa (q := sc oSA4) (by decide)]; exact T.sx.lay.stkD i1),
+ ce_bytesAt' (by decide) (by rw [keep_rsp k2]; exact T.sy.lay.stkD i1), hm1, hm2, e]
+
+end
+
/-! ## Moves with immediates -/
theorem w32_toNat {v : Nat} (h : v < 2 ^ 32) : (BitVec.setWidth 32 (BitVec.ofNat 64 v)).toNat = v := by
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean
index 0713c4074..85c4fa302 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean
@@ -16,8 +16,9 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Same
# ML-DSA key generation on x86-64, with this library's primitives
Untrusted: everything here is checked by Lean. The x86-64 implementations of
-the primitives (`prims`) are verified, use at most 16 bytes of stack, and
-never write `rsp` but by calls nested at most twice, with any implementation
+the primitives (`prims`) are verified, use at most 16 bytes of stack (24
+for `vg_mldsa_rej_ntt_poly4`), and never write `rsp` but by calls nested at
+most twice (three times for `vg_mldsa_rej_ntt_poly4`), with any implementation
`v` of the polynomial arithmetic (`prims_okWith`), so key generation with
them is verified (`keyGen_verifiedWith`).
-/
@@ -50,6 +51,7 @@ theorem prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) where
by decide +kernel⟩ : Callee prims.simpleBitPack _)
bitPack := (⟨⟨0, by decide, Pack.bitPack_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ :
Callee prims.bitPack _)
+ rej4 := ⟨v.ok.rej4.ver, v.ok.rej4.nosp, v.ok.rej4.depth⟩
/-! For any implementation of the polynomial arithmetic, that key
generation never writes `rsp` is checked by evaluating it with every
@@ -57,7 +59,7 @@ function of it empty (`keyGen_same`, as `sign_same`). -/
theorem keyGen_same {m mc : Prog isa → Bool} (hm : Comp m mc) {B : Backend} (h1 : mc B.ntt = true)
(h2 : mc B.invNtt = true) (h3 : mc B.mul = true) (h4 : mc B.mulAdd = true) (h5 : mc B.add = true)
- (p : Spec.MlDsa.Params) : Same m (keyGen (primsWith B) p) (keyGen (primsWith .empty) p) := by
+ (h6 : mc B.rej4 = true) (p : Spec.MlDsa.Params) : Same m (keyGen (primsWith B) p) (keyGen (primsWith .empty) p) := by
unfold keyGen
same_tac hm
@@ -73,7 +75,8 @@ include hp
theorem keyGen_spSafe : (keyGen (primsWith v.code) p).all (fun i => !isa.writesSp i) = true :=
Code.all_of_allInstrs (Same.ok (keyGen_same (Comp.all _) (Code.allInstrs_of_all v.ok.ntt.sp)
(Code.allInstrs_of_all v.ok.invNtt.sp) (Code.allInstrs_of_all v.ok.mul.sp)
- (Code.allInstrs_of_all v.ok.mulAdd.sp) (Code.allInstrs_of_all v.ok.add.sp) p) (keyGen0_sp hp))
+ (Code.allInstrs_of_all v.ok.mulAdd.sp) (Code.allInstrs_of_all v.ok.add.sp)
+ (Code.allInstrs_of_all v.ok.rej4.sp) p) (keyGen0_sp hp))
theorem keyGen_verifiedWith :
Verified X86_64.target (keyGen (primsWith v.code) p) (Spec.MlDsa.keyGenContract p X86_64.abi 32) :=
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inv.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inv.lean
index 4dd5e19c1..24f2939c2 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inv.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inv.lean
@@ -43,7 +43,9 @@ def kcChk (p : Params) (ws : List (Ptr × Nat)) : Bool := topChk (kgB p) ws && k
/-- A piece that writes `ws` keeps `K1`. -/
def k1Chk (p : Params) (ws : List (Ptr × Nat)) : Bool :=
kcChk p ws && keepB (kgB p) ws (sc oHX) 128 && keepB (kgB p) ws (sc oSA) 32 && keepB (kgB p) ws (sc oSB) 64 &&
- keepB (kgB p) ws (sc (oSB + 65)) 1
+ keepB (kgB p) ws (sc (oSB + 65)) 1 && keepB (kgB p) ws (sc (oSA4 + 34 * 0)) 32 &&
+ keepB (kgB p) ws (sc (oSA4 + 34 * 1)) 32 && keepB (kgB p) ws (sc (oSA4 + 34 * 2)) 32 &&
+ keepB (kgB p) ws (sc (oSA4 + 34 * 3)) 32
section
variable {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ)
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Lay.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Lay.lean
index 93212f2a2..9c0a9b49d 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Lay.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Lay.lean
@@ -42,7 +42,7 @@ abbrev scrLen (p : Params) : Nat := scratchWords p * 8
/-- `vg_mldsa*_keygen(seed = rdi, pk = rsi, sk = rdx, scratch = rcx) -> eax`, with 32 bytes of stack. -/
def kgK (p : Params) : Contract isa where
pre s :=
- 24 ≤ (s.gpr .rsp).toNat ∧
+ 32 ≤ (s.gpr .rsp).toNat ∧
s.rd = [⟨s.gpr .rdi, 32⟩] ∧ s.wr = [⟨s.gpr .rsi, p.pkLen⟩, ⟨s.gpr .rdx, p.skLen⟩, ⟨s.gpr .rcx, scrLen p⟩] ∧
Region.Disjoint ⟨s.gpr .rdi, 32⟩ ⟨s.gpr .rsi, p.pkLen⟩ ∧ Region.Disjoint ⟨s.gpr .rdi, 32⟩ ⟨s.gpr .rdx, p.skLen⟩ ∧
Region.Disjoint ⟨s.gpr .rdi, 32⟩ ⟨s.gpr .rcx, scrLen p⟩ ∧
@@ -188,6 +188,7 @@ macro_rules
set_option linter.unusedSimpArgs false in
try simp only [VG.Proof.MlDsa.X86_64.KeyGen.scrLen, VG.Spec.MlDsa.scratchWords,
VG.Impl.MlDsa.X86_64.KeyGen.oP, VG.Impl.MlDsa.X86_64.KeyGen.oSA, VG.Impl.MlDsa.X86_64.KeyGen.oSB,
+ VG.Impl.MlDsa.X86_64.KeyGen.oSA4, VG.Impl.MlDsa.X86_64.KeyGen.oR4,
VG.Impl.MlDsa.X86_64.KeyGen.oHX, VG.Impl.MlDsa.X86_64.KeyGen.oKL, VG.Impl.MlKem.X86_64.oSS,
VG.Impl.MlKem.X86_64.oSV, VG.Impl.MlDsa.X86_64.KeyGen.oT0, $ls,*]
and_intros <;> omega_arith))
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Mask.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Mask.lean
index 2a071d76a..06af58f26 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Mask.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Mask.lean
@@ -19,7 +19,8 @@ open VG.Spec.MlDsa (Params coeffAt)
/-! ## Coefficients in memory -/
-theorem coeffAt_writeW32 (m : Mem) (q : Addr) {i j : Nat} (hi : i < 256) (hj : j < 256) (v : BitVec 32) :
+theorem coeffAt_writeW32 (m : Mem) (q : Addr) {N i j : Nat} (hN : N ≤ 2 ^ 20) (hi : i < N) (hj : j < N)
+ (v : BitVec 32) :
coeffAt (m.writeW (q + BitVec.ofNat 64 (4 * j)) v) q i = if j = i then v else coeffAt m q i := by
unfold coeffAt
split
@@ -50,38 +51,39 @@ theorem and_mask {r : BitVec 32} (hr : r = 0 ∨ r = 1) (x : BitVec 32) :
· rw [ifp rfl, show (BitVec.setWidth 64 (0 - (1 : BitVec 32))).setWidth 32 = BitVec.allOnes 32 by decide]
exact BitVec.and_allOnes
-abbrev maskPre (a : Ptr) : List Instr :=
- [.alu32 .and .r15 (.reg .rax), .mov32 .r8 (.imm 0), .alu32 .sub .r8 (.reg .rax)] ++ lea .rdi a ++ imm .rcx 256
+abbrev maskPre (a : Ptr) (N : Nat) : List Instr :=
+ [.alu32 .and .r15 (.reg .rax), .mov32 .r8 (.imm 0), .alu32 .sub .r8 (.reg .rax)] ++ lea .rdi a ++ imm .rcx N
-theorem maskPre_ok {a : Ptr} (ha : PtrOk a) (h15 : a.1 ≠ .r15) (s : State) :
- WP isa (.block (maskPre a)) s fun s' =>
+theorem maskPre_ok {a : Ptr} (ha : PtrOk a) (h15 : a.1 ≠ .r15) {N : Nat} (hN : N < 2 ^ 32) (s : State) :
+ WP isa (.block (maskPre a N)) s fun s' =>
(s'.mem = s.mem ∧ s'.gpr .r15 = BitVec.setWidth 64 ((s.gpr .r15).setWidth 32 &&& (s.gpr .rax).setWidth 32) ∧
s'.gpr .r8 = BitVec.setWidth 64 (0 - (s.gpr .rax).setWidth 32) ∧ s'.gpr .rdi = pa s a ∧
- s'.gpr .rcx = BitVec.ofNat 64 256) ∧ Keep [.r15, .r8, .rdi, .rcx] s s' := by
+ s'.gpr .rcx = BitVec.ofNat 64 N) ∧ Keep [.r15, .r8, .rdi, .rcx] s s' := by
refine WP.keep _ ?_ (by rfl)
unfold maskPre lea imm
- xrun [sx_ofNat ha.off, imm_eq (show 256 < 2 ^ 32 by decide), h15, ha.ne (r := .r8) (by decide),
+ xrun [sx_ofNat ha.off, imm_eq hN, h15, ha.ne (r := .r8) (by decide),
ha.ne (r := .rdi) (by decide), List.cons_append, List.nil_append]
-theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha : PtrOk a) (ha15 : a.1 ≠ .r15)
- (w : inB (kgW p) a 1024 = true) (hr : (s.gpr .rax).setWidth 32 = 0 ∨ (s.gpr .rax).setWidth 32 = 1) :
- WP isa (mask a) s fun s' => PostB s s' [⟨pa s a, 1024⟩] ∧ MX s' = MX s ∧
+theorem maskN_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha : PtrOk a) (ha15 : a.1 ≠ .r15)
+ {N : Nat} (hN0 : 0 < N) (hN : N ≤ 2 ^ 20) (w : inB (kgW p) a (4 * N) = true)
+ (hr : (s.gpr .rax).setWidth 32 = 0 ∨ (s.gpr .rax).setWidth 32 = 1) :
+ WP isa (mask a N) s fun s' => PostB s s' [⟨pa s a, 4 * N⟩] ∧ MX s' = MX s ∧
s'.gpr .r15 = BitVec.setWidth 64 ((s.gpr .r15).setWidth 32 &&& (s.gpr .rax).setWidth 32) ∧
- ∀ i < 256, coeffAt s'.mem (pa s a) i =
+ ∀ i < N, coeffAt s'.mem (pa s a) i =
if (s.gpr .rax).setWidth 32 = 1 then coeffAt s.mem (pa s a) i else 0 := by
- have hW : InRegions s.wr (pa s a) 1024 := L.cW w _ _ ⟨_, List.mem_singleton_self _, Region.contains_self _ _⟩
- have hn : (pa s a).toNat + 1024 ≤ 2 ^ 64 := L.nwp (inB_mono w)
- refine WP.mono (WP.mx (c := mask a) (noLd_spec (by rfl)) (Q := fun s' => PostB s s' [⟨pa s a, 1024⟩] ∧
+ have hW : InRegions s.wr (pa s a) (4 * N) := L.cW w _ _ ⟨_, List.mem_singleton_self _, Region.contains_self _ _⟩
+ have hn : (pa s a).toNat + 4 * N ≤ 2 ^ 64 := L.nwp (inB_mono w)
+ refine WP.mono (WP.mx (c := mask a N) (noLd_spec (by rfl)) (Q := fun s' => PostB s s' [⟨pa s a, 4 * N⟩] ∧
s'.gpr .r15 = BitVec.setWidth 64 ((s.gpr .r15).setWidth 32 &&& (s.gpr .rax).setWidth 32) ∧
- ∀ i < 256, coeffAt s'.mem (pa s a) i =
+ ∀ i < N, coeffAt s'.mem (pa s a) i =
if (s.gpr .rax).setWidth 32 = 1 then coeffAt s.mem (pa s a) i else 0) ?_)
fun s' ⟨⟨hP, h15, hc⟩, hx⟩ => ⟨hP, hx, h15, hc⟩
unfold mask
- refine WP.seq (WP.mono (maskPre_ok ha ha15 s) fun s1 ⟨⟨hm1, h15, h8, hdi, hcx⟩, k1⟩ => ?_)
- refine WP.mono (wp_countdown (cnt := .rcx) (N := 256) (by decide) (by decide) (fun i s' =>
+ refine WP.seq (WP.mono (maskPre_ok ha ha15 (N := N) (by omega) s) fun s1 ⟨⟨hm1, h15, h8, hdi, hcx⟩, k1⟩ => ?_)
+ refine WP.mono (wp_countdown (cnt := .rcx) (N := N) (by omega) hN0 (fun i s' =>
s'.gpr .rdi = pa s a + BitVec.ofNat 64 (4 * i) ∧ s'.gpr .r8 = s1.gpr .r8 ∧ s'.gpr .r15 = s1.gpr .r15 ∧
- s'.rd = s.rd ∧ s'.wr = s.wr ∧ Frame [⟨pa s a, 1024⟩] s.mem s'.mem ∧
- (∀ j < 256, coeffAt s'.mem (pa s a) j =
+ s'.rd = s.rd ∧ s'.wr = s.wr ∧ Frame [⟨pa s a, 4 * N⟩] s.mem s'.mem ∧
+ (∀ j < N, coeffAt s'.mem (pa s a) j =
if j < i then coeffAt s.mem (pa s a) j &&& (s1.gpr .r8).setWidth 32 else coeffAt s.mem (pa s a) j) ∧
Keep [.r15, .r8, .rdi, .rcx, .rax] s s')
(fun i hi s' ⟨hdi', h8', h15', hrd', hwr', hf, hc, kk⟩ _ => ?_) (fun _ h => h)
@@ -89,7 +91,7 @@ theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha :
fun j _ => by rw [ifn (Nat.not_lt_zero j), hm1], k1.mono (by decide)⟩ hcx)
fun s' ⟨_, h8', h15', hrd', hwr', hf, hc, kk⟩ => ⟨⟨hrd', hwr', fun r hr => kk.gpr ?_, kk.gpr (by decide),
hf.mono fun _ hr => List.mem_append_left _ hr⟩, ?_, ?_⟩
- · have hc4 : (⟨pa s a, 1024⟩ : Region).Contains (pa s a + BitVec.ofNat 64 (4 * i)) 4 :=
+ · have hc4 : (⟨pa s a, 4 * N⟩ : Region).Contains (pa s a + BitVec.ofNat 64 (4 * i)) 4 :=
contains_offset' (by omega) (by omega)
have hin : InRegions s'.wr (s'.gpr .rdi) 4 := by
rw [hwr', hdi']; exact inRegions_sub hW (by omega) (by omega)
@@ -101,7 +103,7 @@ theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha :
(kk.trans k').mono (by decide)⟩, hcx'', hz⟩
· rw [hdi'', hdi', show (4 : BitVec 64) = BitVec.ofNat 64 4 from rfl, off_add]; rfl
· rw [hm, hdi']; exact hf.writeW (List.mem_singleton_self _) _ hc4
- · rw [hm, hdi', coeffAt_writeW32 _ _ hj (by omega), h8']
+ · rw [hm, hdi', coeffAt_writeW32 _ _ hN hj (by omega), h8']
by_cases e : i = j
· subst e
rw [ifp rfl, ifp (Nat.lt_succ_self _)]
@@ -117,6 +119,14 @@ theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha :
· rw [h15', h15]
· intro j hj
rw [hc j hj, ifp hj, h8, and_mask hr]
+theorem mask_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a : Ptr} (ha : PtrOk a) (ha15 : a.1 ≠ .r15)
+ (w : inB (kgW p) a 1024 = true) (hr : (s.gpr .rax).setWidth 32 = 0 ∨ (s.gpr .rax).setWidth 32 = 1) :
+ WP isa (mask a) s fun s' => PostB s s' [⟨pa s a, 1024⟩] ∧ MX s' = MX s ∧
+ s'.gpr .r15 = BitVec.setWidth 64 ((s.gpr .r15).setWidth 32 &&& (s.gpr .rax).setWidth 32) ∧
+ ∀ i < 256, coeffAt s'.mem (pa s a) i =
+ if (s.gpr .rax).setWidth 32 = 1 then coeffAt s.mem (pa s a) i else 0 :=
+ maskN_ok L ha ha15 (N := 256) (by decide) (by decide) w hr
+
/-! ## Constant time -/
/-- The check is the same for every offset: its hint is computed once, for offset 0. -/
@@ -128,4 +138,13 @@ theorem mask_tr {j : Nat} (hj : j < 128) {P : State → State → Prop} (h : ∀
taintRel [.rbx] (fun x y hp r hr => by simp only [List.mem_singleton] at hr; subst hr; exact h x y hp)
(mask_taint j hj)
+/-- The same for the four polynomials from `oP j`. -/
+theorem mask4_taint : ∀ j < 128, (taint.check (X86_64.Taint.ofRegs [.rbx]) (mask (sc (oP j)) 1024)
+ (VG.Taint.hintOf taint (X86_64.Taint.ofRegs [.rbx]) (mask (sc 0) 1024))).isSome = true := by decide +kernel
+
+theorem mask4_tr {j : Nat} (hj : j < 128) {P : State → State → Prop} (h : ∀ x y, P x y → x.gpr .rbx = y.gpr .rbx) :
+ RelCT isa P (mask (sc (oP j)) 1024) fun _ _ => True :=
+ taintRel [.rbx] (fun x y hp r hr => by simp only [List.mem_singleton] at hr; subst hr; exact h x y hp)
+ (mask4_taint j hj)
+
end VG.Proof.MlDsa.X86_64.KeyGen
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestBase.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestBase.lean
index d8133e70a..ecc0c8bd2 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestBase.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestBase.lean
@@ -1,4 +1,4 @@
-import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Samp
+import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Samp4
import VerifiedGarbage.Proof.MlDsa.KeyGen.Rest
/-!
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp.lean
index 7fa5dc6c2..e66a27665 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp.lean
@@ -202,8 +202,8 @@ theorem rej_pub {p : Params} {σ₁ σ₂ : State} (pub : (kgK p).pub σ₁ σ
theorem Two.post {p : Params} {x y x' y' : State} (T : Two p x y) {W₁ W₂ : List Region} (hx : PostB x x' W₁)
(hy : PostB y y' W₂) : Two p x' y' :=
- ⟨⟨T.sx.lay.post hx (kgB_bases p), by rw [hx.rsp]; exact T.sx.h24⟩,
- ⟨T.sy.lay.post hy (kgB_bases p), by rw [hy.rsp]; exact T.sy.h24⟩,
+ ⟨⟨T.sx.lay.post hx (kgB_bases p), by rw [hx.rsp]; exact T.sx.h32⟩,
+ ⟨T.sy.lay.post hy (kgB_bases p), by rw [hy.rsp]; exact T.sy.h32⟩,
fun r hr => by
have hb : r ∈ bases := by simp only [kgRegs, List.mem_cons, List.not_mem_nil, or_false] at hr
rcases hr with rfl | rfl | rfl | rfl <;> decide
@@ -400,14 +400,6 @@ theorem KSamp.zero {p : Params} {σ s : State} (h : K1 p σ s) (h15 : s.gpr .r15
fun _ h => absurd h (Nat.not_lt_zero _),
.inl ⟨h15, ⟨0, 0, 0, 0⟩, fun _ h => absurd h (Nat.not_lt_zero _), fun _ h => absurd h (Nat.not_lt_zero _)⟩⟩
-/-- The entries of `Â`. -/
-theorem sampA_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) :
- Piece p (fun σ s => K1 p σ s ∧ s.gpr .r15 = 1) (fun σ s => KSamp p σ (p.k * p.ℓ) 0 s)
- (seqR (expA P p) 0 (p.k * p.ℓ)) := by
- refine Piece.mono (Piece.seqR (I := fun e σ s => KSamp p σ e 0 s) (p.k * p.ℓ) 0
- fun e _ he => expA_piece hP hF (by omega)) (fun σ s _ h => KSamp.zero h.1 h.2) fun σ s _ h => ?_
- simpa using h
-
/-- The entries of `s₁ ‖ s₂`. -/
theorem sampS_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) :
Piece p (fun σ s => KSamp p σ (p.k * p.ℓ) 0 s) (fun σ s => KSamp p σ (p.k * p.ℓ) (p.ℓ + p.k) s)
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp4.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp4.lean
new file mode 100644
index 000000000..aab0228cc
--- /dev/null
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Samp4.lean
@@ -0,0 +1,208 @@
+import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Samp
+
+/-!
+# ML-DSA key generation on x86-64: four entries of `Â` at a time
+
+Untrusted: everything here is checked by Lean. `expA4 g` sets the indices of
+entries `4g, …, 4g + 3` of `Â` in the four seeds at `oSA4` (`slot_piece`),
+samples the four polynomials with one call of `vg_mldsa_rej_ntt_poly4`, and
+masks them with its result (`call_piece`): it takes `KSamp (4g)` to
+`KSamp (4g + 4)` (`expA4_piece`), and `expAll`, the groups and then the
+last `kℓ mod 4` entries one at a time, takes `K1` to `KSamp (kℓ)`
+(`sampAll_piece`).
+-/
+
+namespace VG.Proof.MlDsa.X86_64.KeyGen
+
+open VG VG.X86_64 VG.Proof.MlKem.X86_64
+open VG.Impl.MlKem.X86_64 (Ptr sc setB seqR)
+open VG.Impl.MlDsa.X86_64.KeyGen
+open VG.Spec.MlDsa (Params Poly rejNTTPoly coeffAt polyAt Reduced PolyIs poly4 seed4 minBounds)
+open VG.Proof.MlDsa.KeyGen (seedA)
+open VG.Spec.Sha3 (bytesAt)
+
+/-! ## The seeds -/
+
+/-- After the first `e` entries of `Â`, with the seeds of entries `e, …, e + j - 1` at `oSA4`. -/
+structure GS (p : Params) (σ : State) (e j : Nat) (s : State) : Prop where
+ ks : KSamp p σ e 0 s
+ done : ∀ k < j, bytesAt s.mem (pa s (sc (oSA4 + 34 * k))) 34 = seedA (rhoOf p σ) ((e + k) / p.ℓ) ((e + k) % p.ℓ)
+
+theorem slot_ok {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) {e j : Nat} (he : e + 4 ≤ p.k * p.ℓ)
+ (hj : j < 4) {s : State} (h : GS p σ e j s) : WP isa (.block (setSR p e j)) s (GS p σ e (j + 1)) := by
+ have hkl := hF.kl; have hl := hF.l; have hk := hF.k
+ have L := h.ks.k1.kc.lay hF hp
+ have hq : (e + j) / p.ℓ < 256 := Nat.lt_of_le_of_lt (Nat.div_le_self _ _) (by omega)
+ have hr : (e + j) % p.ℓ < 256 := Nat.lt_of_lt_of_le (Nat.mod_lt _ (by omega)) (by omega)
+ unfold setSR
+ refine WP.mono (setTwo_ok L (o := oSA4 + 34 * j + 32) hr hq (by lay) (by lay) (by lay))
+ fun s' ⟨hP, hx, hb⟩ => ⟨h.ks.keep hF hp hP.b hx (by layk) (fun e' he' => by layk)
+ (fun _ h => absurd h (Nat.not_lt_zero _)) (hP.cs .r15 (by decide)), fun k hk => ?_⟩
+ rcases (by omega : k < j ∨ k = j) with hk' | rfl
+ · rw [L.keepBytes hP.b (by layk)]; exact h.done k hk'
+ · rw [show 34 = 32 + 2 from rfl, Proof.MlKem.bytesAt_add, L.keepBytes hP.b (by layk), h.ks.k1.sa4 k hj,
+ show pa s' (sc (oSA4 + 34 * k)) + BitVec.ofNat 64 32 = pa s' (sc (oSA4 + 34 * k + 32)) from off_add _ _ _,
+ hP.pa rbx_cs, hb, seedA_eq]
+
+theorem slot_piece {p : Params} (hF : PFacts p) {e j : Nat} (he : e + 4 ≤ p.k * p.ℓ) (hj : j < 4) :
+ Piece p (GS p · e j) (GS p · e (j + 1)) (.block (setSR p e j)) :=
+ ⟨fun _ _ hp h => slot_ok hF hp he hj h,
+ rel_of (Q := Two p) (taintRel [.rbx] (fun x y h => two_rbx h) (hc := .block []) (by with_unfolding_all rfl))
+ fun _ _ _ _ p₁ p₂ pub h₁ h₂ => kc_two hF p₁ p₂ pub h₁.ks.k1.kc h₂.ks.k1.kc⟩
+
+theorem seed4_eq {p : Params} {σ : State} {e : Nat} {s : State} (h : GS p σ e 4 s) {k : Nat} (hk : k < 4) :
+ seed4 s.mem (pa s (sc oSA4)) k = seedA (rhoOf p σ) ((e + k) / p.ℓ) ((e + k) % p.ℓ) := by
+ unfold seed4
+ rw [show pa s (sc oSA4) + BitVec.ofNat 64 (34 * k) = pa s (sc (oSA4 + 34 * k)) from off_add _ _ _]
+ exact h.done k hk
+
+theorem bytes136 (m : Mem) (P : Addr) : bytesAt m P 136 = bytesAt m P 34 ++ bytesAt m (P + BitVec.ofNat 64 34) 34 ++
+ bytesAt m (P + BitVec.ofNat 64 68) 34 ++ bytesAt m (P + BitVec.ofNat 64 102) 34 := by
+ rw [show 136 = 34 + 102 from rfl, Proof.MlKem.bytesAt_add, show 102 = 34 + 68 from rfl, Proof.MlKem.bytesAt_add,
+ show 68 = 34 + 34 from rfl, Proof.MlKem.bytesAt_add]
+ simp only [BitVec.add_assoc, ← BitVec.ofNat_add, List.append_assoc, Nat.reduceAdd]
+
+/-- The four seeds are those of the entries, from `ρ`. -/
+theorem GS.seeds {p : Params} {σ : State} {e : Nat} {s : State} (h : GS p σ e 4 s) :
+ bytesAt s.mem (pa s (sc oSA4)) 136 = seedA (rhoOf p σ) ((e + 0) / p.ℓ) ((e + 0) % p.ℓ) ++
+ seedA (rhoOf p σ) ((e + 1) / p.ℓ) ((e + 1) % p.ℓ) ++ seedA (rhoOf p σ) ((e + 2) / p.ℓ) ((e + 2) % p.ℓ) ++
+ seedA (rhoOf p σ) ((e + 3) / p.ℓ) ((e + 3) % p.ℓ) := by
+ have b : ∀ k < 4, bytesAt s.mem (pa s (sc oSA4) + BitVec.ofNat 64 (34 * k)) 34 =
+ seedA (rhoOf p σ) ((e + k) / p.ℓ) ((e + k) % p.ℓ) := fun k hk => seed4_eq h hk
+ have b0 := b 0 (by decide)
+ rw [show 34 * 0 = 0 from rfl, add_ofNat_zero] at b0
+ rw [bytes136, b0, b 1 (by decide), b 2 (by decide), b 3 (by decide)]
+
+/-! ## The four polynomials -/
+
+theorem coeffAt_poly4 (m : Mem) (a : Addr) (k j : Nat) : coeffAt m (poly4 a k) j = coeffAt m a (256 * k + j) := by
+ unfold coeffAt poly4
+ rw [BitVec.add_assoc, ← BitVec.ofNat_add, show 1024 * k + 4 * j = 4 * (256 * k + j) by omega]
+
+theorem pa_poly4 (s : State) (e k : Nat) : poly4 (pa s (aP e)) k = pa s (aP (e + k)) := by
+ unfold poly4
+ rw [show pa s (aP e) + BitVec.ofNat 64 (1024 * k) = pa s (sc (oP e + 1024 * k)) from off_add _ _ _,
+ show oP e + 1024 * k = oP (e + k) by simp only [oP]; omega]
+
+/-- One bound for the four polynomials. -/
+theorem bound4 {ρ : Nat → List Byte} {x : Nat → Poly} (h : ∀ k < 4, ∃ b : Spec.MlDsa.Bounds,
+ rejNTTPoly b.rejNTT (ρ k) = some (x k)) : ∃ n : Nat, ∀ k < 4, rejNTTPoly n (ρ k) = some (x k) := by
+ obtain ⟨b0, h0⟩ := h 0 (by decide)
+ obtain ⟨b1, h1⟩ := h 1 (by decide)
+ obtain ⟨b2, h2⟩ := h 2 (by decide)
+ obtain ⟨b3, h3⟩ := h 3 (by decide)
+ refine ⟨max (max b0.rejNTT b1.rejNTT) (max b2.rejNTT b3.rejNTT), fun k hk => ?_⟩
+ rcases (by omega : k = 0 ∨ k = 1 ∨ k = 2 ∨ k = 3) with rfl | rfl | rfl | rfl
+ · exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (by omega) h0
+ · exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (by omega) h1
+ · exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (by omega) h2
+ · exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (by omega) h3
+
+theorem call_ok {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ)
+ {g : Nat} (hg : 4 * g + 4 ≤ p.k * p.ℓ) {s : State} (h : GS p σ (4 * g) 4 s) :
+ WP isa (.seq (rej4At P.rej4 P.sfx (aP (4 * g)) (sc (oR4 p))) (mask (aP (4 * g)) 1024)) s
+ (KSamp p σ (4 * g + 4) 0) := by
+ have hkl := hF.kl; have hl := hF.l; have hk := hF.k
+ have S := h.ks.k1.kc.site hF hp
+ have L := S.lay
+ refine WP.seq (WP.mono (rej4At_ok (a := aP (4 * g)) (w := sc (oR4 p)) (sc_ok _ (by simp only [oP]; omega))
+ (sc_ok _ (by simp only [oR4, oP]; omega)) (by lay) (by lay) (by lay) (by lay) (by lay) hP.rej4 S)
+ fun s₂ ⟨hP₂, hx₂, hred, hout⟩ => ?_)
+ have hP₂' : PPostB s s₂ [(aP (4 * g), 4096), (sc (oR4 p), 8192)] := hP₂.b
+ have L₂ := L.post hP₂' (kgB_bases p)
+ have hr01 : (s₂.gpr .rax).setWidth 32 = 0 ∨ (s₂.gpr .rax).setWidth 32 = 1 := by
+ rcases hout with ⟨h1, _⟩ | ⟨h0, _⟩
+ exacts [.inr h1, .inl h0]
+ refine WP.mono (maskN_ok L₂ (a := aP (4 * g)) (N := 1024) (sc_ok _ (by simp only [oP]; omega))
+ (show Reg.rbx ≠ .r15 by decide) (by decide) (by decide) (by lay) hr01) fun s₃ ⟨hP₃, hx₃, h15, hco⟩ => ?_
+ have hP₃' : PPostB s₂ s₃ [(aP (4 * g), 4 * 1024)] := hP₃
+ have hP₁₃ := PPostB.app hP₂' hP₃' (sc1_bases _ _)
+ have e₂ : pa s₂ (aP (4 * g)) = pa s (aP (4 * g)) := hP₂'.pa rbx_bases
+ rw [e₂] at hco
+ have hco4 : ∀ k < 4, ∀ i < 256, coeffAt s₃.mem (poly4 (pa s (aP (4 * g))) k) i =
+ if (s₂.gpr .rax).setWidth 32 = 1 then coeffAt s₂.mem (poly4 (pa s (aP (4 * g))) k) i else 0 :=
+ fun k hk i hi => by rw [coeffAt_poly4, coeffAt_poly4]; exact hco _ (by omega)
+ -- Entry `4g + k` is polynomial `k` from `aP (4g)`.
+ have e₃ : ∀ k, pa s₃ (aP (4 * g + k)) = poly4 (pa s (aP (4 * g))) k := fun k => by
+ rw [pa_poly4]; exact hP₁₃.pa rbx_bases
+ obtain ⟨A, S', hA, _, hG⟩ := h.ks.ex
+ have h15₂ : s₂.gpr .r15 = s.gpr .r15 := hP₂.cs .r15 (by decide)
+ rw [h15₂, r15_and (good_01 hG) hr01] at h15
+ refine ⟨h.ks.k1.step hF hp hP₁₃ (hx₃.trans hx₂) (by layk), fun e' => if e' < 4 * g then A e'
+ else polyAt s₃.mem (pa s₃ (aP e')), S', fun e' he' => ?_, fun _ h => absurd h (Nat.not_lt_zero _), ?_⟩
+ · dsimp only
+ by_cases hlt : e' < 4 * g
+ · rw [ifp hlt]
+ exact polyIs_frame' L hP₁₃ (by layk) (hA e' hlt)
+ · rw [ifn hlt]
+ refine ⟨?_, rfl⟩
+ obtain ⟨k, rfl⟩ : ∃ k, e' = 4 * g + k := ⟨e' - 4 * g, by omega⟩
+ rw [e₃]
+ by_cases h1 : (s₂.gpr .rax).setWidth 32 = 1
+ · exact (masked_one h1 (hco4 k (by omega))).2 (hred h1 k (by omega))
+ · exact (masked_zero h1 (hco4 k (by omega))).1
+ · rw [h15]
+ rcases hG with ⟨h1, b, hb, _⟩ | ⟨h0, hn⟩
+ · rcases hout with ⟨ho, hb4⟩ | ⟨ho, k, hk, hn⟩
+ · obtain ⟨n, hn⟩ := bound4 hb4
+ refine .inl ⟨by rw [ifp ⟨h1, ho⟩], { b with rejNTT := max b.rejNTT n }, fun e' he' => ?_,
+ fun _ h => absurd h (Nat.not_lt_zero _)⟩
+ dsimp only
+ by_cases hlt : e' < 4 * g
+ · rw [ifp hlt]
+ exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (Nat.le_max_left _ _) (hb e' hlt)
+ · rw [ifn hlt]
+ obtain ⟨k, rfl⟩ : ∃ k, e' = 4 * g + k := ⟨e' - 4 * g, by omega⟩
+ have hk : k < 4 := by omega
+ rw [e₃, (masked_one ho (hco4 k hk)).1, ← seed4_eq h hk]
+ exact Proof.MlDsa.KeyGen.rejNTTPoly_mono (Nat.le_max_right _ _) (hn k hk)
+ · rw [ifn (fun h => by rw [h.2] at ho; exact absurd ho (by decide))]
+ rw [seed4_eq h hk] at hn
+ have hq : (4 * g + k) / p.ℓ < p.k := Nat.div_lt_of_lt_mul (by rw [Nat.mul_comm p.ℓ p.k]; omega)
+ exact .inr ⟨rfl, Proof.MlDsa.KeyGen.keyGenInternal_none_A hq (Nat.mod_lt _ (by omega)) hn⟩
+ · rw [ifn (fun h => by rw [h.1] at h0; exact absurd h0 (by decide))]
+ exact .inr ⟨rfl, hn⟩
+
+theorem call_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) {g : Nat}
+ (hg : 4 * g + 4 ≤ p.k * p.ℓ) :
+ Piece p (GS p · (4 * g) 4) (KSamp p · (4 * g + 4) 0)
+ (.seq (rej4At P.rej4 P.sfx (aP (4 * g)) (sc (oR4 p))) (mask (aP (4 * g)) 1024)) := by
+ have hkl := hF.kl; have hl := hF.l; have hk := hF.k
+ refine ⟨fun _ _ hp h => call_ok hP hF hp hg h, rel_of (Q := fun x y => Two p x y ∧
+ bytesAt x.mem (pa x (sc oSA4)) 136 = bytesAt y.mem (pa y (sc oSA4)) 136) ?_
+ fun _ _ _ _ p₁ p₂ pub h₁ h₂ => ⟨kc_two hF p₁ p₂ pub h₁.ks.k1.kc h₂.ks.k1.kc,
+ by rw [h₁.seeds, h₂.seeds, rho_pub pub]⟩⟩
+ have ok := fun x (S : Site p x) => WP.mono (rej4At_ok (a := aP (4 * g)) (w := sc (oR4 p)) (sfx := P.sfx)
+ (sc_ok _ (by simp only [oP]; omega)) (sc_ok _ (by simp only [oR4, oP]; omega)) (by lay) (by lay) (by lay)
+ (by lay) (by lay) hP.rej4 S) fun _ h => (⟨_, h.1.b⟩ : ∃ W, PostB x _ W)
+ exact RelCT.seq (RelCT.two (fun _ _ h => h.1) (rej4At_tr (sfx := P.sfx) (sc_ok _ (by simp only [oP]; omega))
+ (sc_ok _ (by simp only [oR4, oP]; omega)) (by lay) (by lay) (by lay) (by lay) (by lay) hP.rej4
+ (show Reg.rbx ∈ kgRegs by decide) (show Reg.rbx ∈ kgRegs by decide))
+ fun x y h => ⟨ok x h.1.sx, ok y h.1.sy⟩)
+ (mask4_tr (j := 4 * g) (by omega) fun x y h => h.regs .rbx (by decide))
+
+/-! ## The pieces -/
+
+theorem expA4_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) {g : Nat}
+ (hg : 4 * g + 4 ≤ p.k * p.ℓ) :
+ Piece p (KSamp p · (4 * g) 0) (KSamp p · (4 * g + 4) 0) (expA4 P p g) := by
+ unfold expA4
+ exact Piece.mono ((slot_piece hF hg (j := 0) (by decide)).seq ((slot_piece hF hg (j := 1) (by decide)).seq
+ ((slot_piece hF hg (j := 2) (by decide)).seq ((slot_piece hF hg (j := 3) (by decide)).seq (call_piece hP hF hg)))))
+ (fun _ _ _ h => ⟨h, fun _ h => absurd h (Nat.not_lt_zero _)⟩) fun _ _ _ h => h
+
+/-- The entries of `Â`. -/
+theorem sampAll_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) :
+ Piece p (fun σ s => K1 p σ s ∧ s.gpr .r15 = 1) (fun σ s => KSamp p σ (p.k * p.ℓ) 0 s) (expAll P p) := by
+ unfold expAll
+ refine Piece.seq (J := fun σ s => KSamp p σ (4 * (p.k * p.ℓ / 4)) 0 s) ?_ ?_
+ · refine Piece.mono (Piece.seqR (I := fun g σ s => KSamp p σ (4 * g) 0 s) (p.k * p.ℓ / 4) 0
+ fun g _ hg => Piece.mono (expA4_piece hP hF (g := g) (by omega)) (fun _ _ _ h => h) fun _ _ _ h => ?_)
+ (fun σ s _ h => KSamp.zero h.1 h.2) fun σ s _ h => ?_
+ · rw [show 4 * (g + 1) = 4 * g + 4 by omega]; exact h
+ · simpa using h
+ · refine Piece.mono (Piece.seqR (I := fun e σ s => KSamp p σ e 0 s) (p.k * p.ℓ % 4) (4 * (p.k * p.ℓ / 4))
+ fun e h1 h2 => expA_piece hP hF (by omega)) (fun _ _ _ h => h) fun σ s _ h => ?_
+ rwa [show 4 * (p.k * p.ℓ / 4) + p.k * p.ℓ % 4 = p.k * p.ℓ by omega] at h
+
+end VG.Proof.MlDsa.X86_64.KeyGen
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Seeds.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Seeds.lean
index cb6aac8a6..b71079899 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Seeds.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Seeds.lean
@@ -28,8 +28,8 @@ theorem Two.lrel {p : Params} {x y : State} (h : Two p x y) : LRel kgR (kgW p) x
theorem Two.step {p : Params} {c : Prog isa} (htr : RelCT isa (Two p) c fun _ _ => True)
(hok : ∀ x, Site p x → WP isa c x fun x' => ∃ W, PostB x x' W) : RelCT isa (Two p) c (Two p) :=
RelCT.postDep htr (F := fun x x' => ∃ W, PostB x x' W) (fun x y h => ⟨hok x h.sx, hok y h.sy⟩)
- fun x y x' y' h ⟨_, hx⟩ ⟨_, hy⟩ => ⟨⟨h.sx.lay.post hx (kgB_bases p), by rw [hx.rsp]; exact h.sx.h24⟩,
- ⟨h.sy.lay.post hy (kgB_bases p), by rw [hy.rsp]; exact h.sy.h24⟩,
+ fun x y x' y' h ⟨_, hx⟩ ⟨_, hy⟩ => ⟨⟨h.sx.lay.post hx (kgB_bases p), by rw [hx.rsp]; exact h.sx.h32⟩,
+ ⟨h.sy.lay.post hy (kgB_bases p), by rw [hy.rsp]; exact h.sy.h32⟩,
fun r hr => by
have hb : r ∈ bases := by simp only [kgRegs, List.mem_cons, List.not_mem_nil, or_false] at hr
rcases hr with rfl | rfl | rfl | rfl <;> decide
@@ -137,21 +137,30 @@ theorem pro_piece {p : Params} (hF : PFacts p) :
/-- `H(ξ ‖ k ‖ ℓ, 128)`. -/
abbrev hxOf (p : Params) (σ : State) : List Byte := Spec.MlDsa.H (xiOf σ ++ integerToBytes p.k 1 ++ integerToBytes p.ℓ 1) 128
-/-- After the seeds. -/
-structure K1 (p : Params) (σ s : State) : Prop where
+/-- After the seeds, with `ρ` copied to the first `j` seeds of `oSA4`. -/
+structure K1R (p : Params) (j : Nat) (σ s : State) : Prop where
kc : KC p σ s
hx : bytesAt s.mem (pa s (sc oHX)) 128 = hxOf p σ
sa : bytesAt s.mem (pa s (sc oSA)) 32 = rhoOf p σ
sb : bytesAt s.mem (pa s (sc oSB)) 64 = rho'Of p σ
z : bytesAt s.mem (pa s (sc (oSB + 65))) 1 = [0]
+ sa4 : ∀ k < j, bytesAt s.mem (pa s (sc (oSA4 + 34 * k))) 32 = rhoOf p σ
+
+/-- After the seeds. -/
+abbrev K1 (p : Params) (σ s : State) : Prop := K1R p 4 σ s
theorem K1.step {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) {s s' : State} (h : K1 p σ s)
{ws : List (Ptr × Nat)} (hP : PPostB s s' ws) (hx : MX s' = MX s) (hc : k1Chk p ws = true) : K1 p σ s' := by
simp only [k1Chk, Bool.and_eq_true] at hc
- obtain ⟨⟨⟨⟨h0, h1⟩, h2⟩, h3⟩, h4⟩ := hc
+ obtain ⟨⟨⟨⟨⟨⟨⟨⟨h0, h1⟩, h2⟩, h3⟩, h4⟩, a0⟩, a1⟩, a2⟩, a3⟩ := hc
have L := h.kc.lay hF hp
- exact ⟨h.kc.step hF hp hP hx h0, by rw [L.keepBytes hP h1]; exact h.hx, by rw [L.keepBytes hP h2]; exact h.sa,
- by rw [L.keepBytes hP h3]; exact h.sb, by rw [L.keepBytes hP h4]; exact h.z⟩
+ refine ⟨h.kc.step hF hp hP hx h0, by rw [L.keepBytes hP h1]; exact h.hx, by rw [L.keepBytes hP h2]; exact h.sa,
+ by rw [L.keepBytes hP h3]; exact h.sb, by rw [L.keepBytes hP h4]; exact h.z, fun k hk => ?_⟩
+ rcases (by omega : k = 0 ∨ k = 1 ∨ k = 2 ∨ k = 3) with rfl | rfl | rfl | rfl
+ · rw [L.keepBytes hP a0]; exact h.sa4 0 hk
+ · rw [L.keepBytes hP a1]; exact h.sa4 1 hk
+ · rw [L.keepBytes hP a2]; exact h.sa4 2 hk
+ · rw [L.keepBytes hP a3]; exact h.sa4 3 hk
theorem shake31' : BitVec.ofNat 8 31 = Spec.Sha3.shakeSuffix := by decide
@@ -192,6 +201,20 @@ theorem setKL_ok {p : Params} {s : State} (L : Lay kgR (kgW p) s) {a b : Nat} (h
bytesAt s'.mem (pa s (sc oKL)) 2 = [BitVec.ofNat 8 a, BitVec.ofNat 8 b] :=
setTwo_ok L ha hb (by lay) (by lay) (by lay)
+/-- `ρ` to seed `j` of `oSA4`. -/
+theorem copyR_ok {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) {j : Nat} (hj : j < 4) {s : State}
+ (h : K1R p j σ s) :
+ WP isa (copy (sc (oSA4 + 34 * j)) (sc oHX) 32) s fun s' => K1R p (j + 1) σ s' ∧ s'.gpr .r15 = s.gpr .r15 := by
+ have hk := hF.k; have hl := hF.l
+ have L := h.kc.lay hF hp
+ refine WP.mono (copy_okM L (dst := sc (oSA4 + 34 * j)) (src := sc oHX) (n := 32) (by decide) (by layk))
+ fun s' ⟨⟨hP, hb⟩, hx⟩ => ⟨⟨h.kc.step hF hp hP.b hx (by layk), by rw [L.keepBytes hP.b (by layk)]; exact h.hx,
+ by rw [L.keepBytes hP.b (by layk)]; exact h.sa, by rw [L.keepBytes hP.b (by layk)]; exact h.sb,
+ by rw [L.keepBytes hP.b (by layk)]; exact h.z, fun k hk' => ?_⟩, hP.cs .r15 (by decide)⟩
+ rcases (by omega : k < j ∨ k = j) with hk' | rfl
+ · rw [L.keepBytes hP.b (by layk)]; exact h.sa4 k hk'
+ · rw [hP.pa rbx_cs, hb, ← Proof.MlKem.bytesAt_take s.mem (pa s (sc oHX)) (show 32 ≤ 128 by decide), h.hx, ← rho_eq]
+
theorem seeds_ok {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ) {s : State} (h : KC p σ s)
(h15 : s.gpr .r15 = 1) : WP isa (seeds p) s fun s' => K1 p σ s' ∧ s'.gpr .r15 = 1 := by
have hk := hF.k; have hl := hF.l
@@ -224,14 +247,21 @@ theorem seeds_ok {p : Params} (hF : PFacts p) {σ : State} (hp : (kgK p).pre σ)
have hsb : bytesAt s₃.mem (pa s₃ (sc (oHX + 32))) 64 = rho'Of p σ := by
rw [rho'_eq, ← hx3, Proof.MlKem.bytesAt_slice _ _ (show 32 + 64 ≤ 128 by decide), pa, pa, off_add]
rw [hsb, ← e₄] at hb₄
- refine WP.mono (WP.mx (noLd_spec (by rfl)) (setB_okL L₄ (p := sc (oSB + 65)) (v := 0) (by decide) (by decide)
- (by lay))) fun s₅ ⟨⟨hP₅, hb₅⟩, hx₅⟩ => ⟨⟨h₄.step hF hp hP₅.b hx₅ (by layk), ?_, ?_, ?_, ?_⟩, ?_⟩
- · rw [L₄.keepBytes hP₅.b (by layk), L₃.keepBytes hP₄.b (by layk)]; exact hx3
- · rw [L₄.keepBytes hP₅.b (by layk), L₃.keepBytes hP₄.b (by layk)]; exact hb₃
- · rw [L₄.keepBytes hP₅.b (by layk)]; exact hb₄
- · rw [hP₅.pa rbx_cs]; exact hb₅
- · rw [hP₅.cs .r15 (by decide), hP₄.cs .r15 (by decide), hP₃.cs .r15 (by decide), hP₂.cs .r15 (by decide),
+ refine WP.seq (WP.mono (WP.mx (noLd_spec (by rfl)) (setB_okL L₄ (p := sc (oSB + 65)) (v := 0) (by decide)
+ (by decide) (by lay))) fun s₅ ⟨⟨hP₅, hb₅⟩, hx₅⟩ => ?_)
+ have h₅ : K1R p 0 σ s₅ := ⟨h₄.step hF hp hP₅.b hx₅ (by layk),
+ by rw [L₄.keepBytes hP₅.b (by layk), L₃.keepBytes hP₄.b (by layk)]; exact hx3,
+ by rw [L₄.keepBytes hP₅.b (by layk), L₃.keepBytes hP₄.b (by layk)]; exact hb₃,
+ by rw [L₄.keepBytes hP₅.b (by layk)]; exact hb₄, by rw [hP₅.pa rbx_cs]; exact hb₅,
+ fun _ h => absurd h (Nat.not_lt_zero _)⟩
+ have f₅ : s₅.gpr .r15 = 1 := by
+ rw [hP₅.cs .r15 (by decide), hP₄.cs .r15 (by decide), hP₃.cs .r15 (by decide), hP₂.cs .r15 (by decide),
hP₁.cs .r15 (by decide), h15]
+ refine WP.seq (WP.mono (copyR_ok hF hp (j := 0) (by decide) h₅) fun s₆ ⟨h₆, f₆⟩ => ?_)
+ refine WP.seq (WP.mono (copyR_ok hF hp (j := 1) (by decide) h₆) fun s₇ ⟨h₇, f₇⟩ => ?_)
+ refine WP.seq (WP.mono (copyR_ok hF hp (j := 2) (by decide) h₇) fun s₈ ⟨h₈, f₈⟩ => ?_)
+ exact WP.mono (copyR_ok hF hp (j := 3) (by decide) h₈) fun s₉ ⟨h₉, f₉⟩ =>
+ ⟨h₉, by rw [f₉, f₈, f₇, f₆, f₅]⟩
theorem setKL_taint : ∀ v < 16, ∀ w < 16, (taint.check (X86_64.Taint.ofRegs [.rbx])
(.block (setB (sc oKL) v ++ setB (sc (oKL + 1)) w)) (.block [])).isSome = true := by decide +kernel
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Top.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Top.lean
index fea8439e0..2f93be7e2 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Top.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Top.lean
@@ -140,7 +140,7 @@ theorem rest_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) :
theorem keyGen_piece {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) :
Piece p (fun σ s => s = σ) (fun σ s => abiPreserved σ s ∧ (kgK p).post σ s) (keyGen P p) :=
- (pro_piece hF).seq ((seeds_piece hF).seq ((sampA_piece hP hF).seq ((sampS_piece hP hF).seq
+ (pro_piece hF).seq ((seeds_piece hF).seq ((sampAll_piece hP hF).seq ((sampS_piece hP hF).seq
((rest_piece hP hF).seq (epi_piece hF)))))
theorem keyGen_correct {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) (σ : State) (hp : (kgK p).pre σ) :
diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs
index 3ac477362..b51c4cdf8 100644
--- a/src/asm/x86_64/mldsa44.rs
+++ b/src/asm/x86_64/mldsa44.rs
@@ -132,119 +132,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"jne 21b",
"mov eax, 0",
"mov BYTE PTR [rbx+1281], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 4096",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 4096",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"22:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 22b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1442",
"mov rsi, rbx",
- "add rsi, 5120",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 5120",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"23:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 23b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1476",
"mov rsi, rbx",
- "add rsi, 6144",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"24:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 24b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1510",
"mov rsi, rbx",
- "add rsi, 7168",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 7168",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"25:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 25b",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 8192",
+ "add rsi, 4096",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 31744",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 8192",
- "mov ecx, 256",
+ "add rdi, 4096",
+ "mov ecx, 1024",
"26:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -252,270 +216,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 26b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 9216",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 9216",
- "mov ecx, 256",
- "27:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 27b",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1508], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 10240",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 10240",
- "mov ecx, 256",
- "28:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 28b",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1542], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 11264",
+ "add rsi, 8192",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 31744",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 11264",
- "mov ecx, 256",
- "29:",
+ "add rdi, 8192",
+ "mov ecx, 1024",
+ "27:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 29b",
+ "jne 27b",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 12288",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 12288",
- "mov ecx, 256",
- "210:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 210b",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 13312",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 13312",
- "mov ecx, 256",
- "211:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 211b",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1508], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 14336",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 14336",
- "mov ecx, 256",
- "212:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 212b",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1542], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 15360",
+ "add rsi, 12288",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 31744",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 15360",
- "mov ecx, 256",
- "213:",
+ "add rdi, 12288",
+ "mov ecx, 1024",
+ "28:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 213b",
+ "jne 28b",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 16384",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 16384",
- "mov ecx, 256",
- "214:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 214b",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 17408",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 17408",
- "mov ecx, 256",
- "215:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 215b",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1508], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 18432",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 18432",
- "mov ecx, 256",
- "216:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 216b",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1542], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 19456",
+ "add rsi, 16384",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 31744",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 19456",
- "mov ecx, 256",
- "217:",
+ "add rdi, 16384",
+ "mov ecx, 1024",
+ "29:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 217b",
+ "jne 29b",
"mov eax, 0",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -532,13 +340,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 20480",
"mov ecx, 256",
- "218:",
+ "210:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 218b",
+ "jne 210b",
"mov eax, 1",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -555,13 +363,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 21504",
"mov ecx, 256",
- "219:",
+ "211:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 219b",
+ "jne 211b",
"mov eax, 2",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -578,13 +386,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 22528",
"mov ecx, 256",
- "220:",
+ "212:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 220b",
+ "jne 212b",
"mov eax, 3",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -601,13 +409,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 23552",
"mov ecx, 256",
- "221:",
+ "213:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 221b",
+ "jne 213b",
"mov eax, 4",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -624,13 +432,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 24576",
"mov ecx, 256",
- "222:",
+ "214:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 222b",
+ "jne 214b",
"mov eax, 5",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -647,13 +455,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 25600",
"mov ecx, 256",
- "223:",
+ "215:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 223b",
+ "jne 215b",
"mov eax, 6",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -670,13 +478,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 26624",
"mov ecx, 256",
- "224:",
+ "216:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 224b",
+ "jne 216b",
"mov eax, 7",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -693,49 +501,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 27648",
"mov ecx, 256",
- "225:",
+ "217:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 225b",
+ "jne 217b",
"mov rdi, r12",
"add rdi, 0",
"mov rsi, rbx",
"add rsi, 1024",
"mov ecx, 32",
- "226:",
+ "218:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 226b",
+ "jne 218b",
"mov rdi, r13",
"add rdi, 0",
"mov rsi, rbx",
"add rsi, 1024",
"mov ecx, 32",
- "227:",
+ "219:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 227b",
+ "jne 219b",
"mov rdi, r13",
"add rdi, 32",
"mov rsi, rbx",
"add rsi, 1120",
"mov ecx, 32",
- "228:",
+ "220:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 228b",
+ "jne 220b",
"mov rdi, rbx",
"add rdi, 20480",
"mov esi, 2",
@@ -1125,7 +933,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32
vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
vg_keccak_pad = sym super::sha3::vg_keccak_pad,
vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
- vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2,
vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
@@ -3390,119 +3198,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"jne 21b",
"mov eax, 0",
"mov BYTE PTR [rbx+1281], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 4096",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 4096",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"22:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 22b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1442",
"mov rsi, rbx",
- "add rsi, 5120",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 5120",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"23:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 23b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1476",
"mov rsi, rbx",
- "add rsi, 6144",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"24:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 24b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1510",
"mov rsi, rbx",
- "add rsi, 7168",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 7168",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"25:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 25b",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 8192",
+ "add rsi, 4096",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 31744",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 8192",
- "mov ecx, 256",
+ "add rdi, 4096",
+ "mov ecx, 1024",
"26:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -3510,270 +3282,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"add rdi, 4",
"sub rcx, 1",
"jne 26b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 9216",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 9216",
- "mov ecx, 256",
- "27:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 27b",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1508], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 10240",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 10240",
- "mov ecx, 256",
- "28:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 28b",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1542], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 11264",
+ "add rsi, 8192",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 31744",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 11264",
- "mov ecx, 256",
- "29:",
+ "add rdi, 8192",
+ "mov ecx, 1024",
+ "27:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 29b",
+ "jne 27b",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 12288",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 12288",
- "mov ecx, 256",
- "210:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 210b",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 13312",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 13312",
- "mov ecx, 256",
- "211:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 211b",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1508], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 14336",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 14336",
- "mov ecx, 256",
- "212:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 212b",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1542], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 15360",
+ "add rsi, 12288",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 31744",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 15360",
- "mov ecx, 256",
- "213:",
+ "add rdi, 12288",
+ "mov ecx, 1024",
+ "28:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 213b",
+ "jne 28b",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 16384",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 16384",
- "mov ecx, 256",
- "214:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 214b",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 17408",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 17408",
- "mov ecx, 256",
- "215:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 215b",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1508], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 18432",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 18432",
- "mov ecx, 256",
- "216:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 216b",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1542], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 19456",
+ "add rsi, 16384",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 31744",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 19456",
- "mov ecx, 256",
- "217:",
+ "add rdi, 16384",
+ "mov ecx, 1024",
+ "29:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 217b",
+ "jne 29b",
"mov eax, 0",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -3790,13 +3406,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 20480",
"mov ecx, 256",
- "218:",
+ "210:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 218b",
+ "jne 210b",
"mov eax, 1",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -3813,13 +3429,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 21504",
"mov ecx, 256",
- "219:",
+ "211:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 219b",
+ "jne 211b",
"mov eax, 2",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -3836,13 +3452,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 22528",
"mov ecx, 256",
- "220:",
+ "212:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 220b",
+ "jne 212b",
"mov eax, 3",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -3859,13 +3475,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 23552",
"mov ecx, 256",
- "221:",
+ "213:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 221b",
+ "jne 213b",
"mov eax, 4",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -3882,13 +3498,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 24576",
"mov ecx, 256",
- "222:",
+ "214:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 222b",
+ "jne 214b",
"mov eax, 5",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -3905,13 +3521,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 25600",
"mov ecx, 256",
- "223:",
+ "215:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 223b",
+ "jne 215b",
"mov eax, 6",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -3928,13 +3544,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 26624",
"mov ecx, 256",
- "224:",
+ "216:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 224b",
+ "jne 216b",
"mov eax, 7",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -3951,49 +3567,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 27648",
"mov ecx, 256",
- "225:",
+ "217:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 225b",
+ "jne 217b",
"mov rdi, r12",
"add rdi, 0",
"mov rsi, rbx",
"add rsi, 1024",
"mov ecx, 32",
- "226:",
+ "218:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 226b",
+ "jne 218b",
"mov rdi, r13",
"add rdi, 0",
"mov rsi, rbx",
"add rsi, 1024",
"mov ecx, 32",
- "227:",
+ "219:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 227b",
+ "jne 219b",
"mov rdi, r13",
"add rdi, 32",
"mov rsi, rbx",
"add rsi, 1120",
"mov ecx, 32",
- "228:",
+ "220:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 228b",
+ "jne 220b",
"mov rdi, rbx",
"add rdi, 20480",
"mov esi, 2",
@@ -4383,7 +3999,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen(seed: *const [u8; 32], pk
vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
vg_keccak_pad = sym super::sha3::vg_keccak_pad,
vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
- vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4,
vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt,
diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs
index f85fb71ee..7d80620cf 100644
--- a/src/asm/x86_64/mldsa65.rs
+++ b/src/asm/x86_64/mldsa65.rs
@@ -132,119 +132,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"jne 21b",
"mov eax, 0",
"mov BYTE PTR [rbx+1281], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 4096",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 4096",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"22:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 22b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1442",
"mov rsi, rbx",
- "add rsi, 5120",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 5120",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"23:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 23b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1476",
"mov rsi, rbx",
- "add rsi, 6144",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"24:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 24b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1510",
"mov rsi, rbx",
- "add rsi, 7168",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 7168",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"25:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 25b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
"mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 8192",
+ "add rsi, 4096",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 8192",
- "mov ecx, 256",
+ "add rdi, 4096",
+ "mov ecx, 1024",
"26:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -252,23 +216,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 26b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 9216",
+ "add rsi, 8192",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 9216",
- "mov ecx, 256",
+ "add rdi, 8192",
+ "mov ecx, 1024",
"27:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -276,23 +252,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 27b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 10240",
+ "add rsi, 12288",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 10240",
- "mov ecx, 256",
+ "add rdi, 12288",
+ "mov ecx, 1024",
"28:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -301,22 +289,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"sub rcx, 1",
"jne 28b",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 11264",
+ "add rsi, 16384",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 11264",
- "mov ecx, 256",
+ "add rdi, 16384",
+ "mov ecx, 1024",
"29:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -324,23 +324,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 29b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 12288",
+ "add rsi, 20480",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 12288",
- "mov ecx, 256",
+ "add rdi, 20480",
+ "mov ecx, 1024",
"210:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -348,23 +360,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 210b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 13312",
+ "add rsi, 24576",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 13312",
- "mov ecx, 256",
+ "add rdi, 24576",
+ "mov ecx, 1024",
"211:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -372,23 +396,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 211b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 14336",
+ "add rsi, 28672",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 14336",
- "mov ecx, 256",
+ "add rdi, 28672",
+ "mov ecx, 1024",
"212:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -396,14 +432,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 212b",
- "mov eax, 1",
+ "mov eax, 3",
"mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
+ "mov eax, 5",
"mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
"add rdi, 1152",
"mov rsi, rbx",
- "add rsi, 15360",
+ "add rsi, 32768",
"mov rdx, rbx",
"add rdx, 2048",
"call {vg_mldsa_rej_ntt_poly}",
@@ -411,7 +447,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 15360",
+ "add rdi, 32768",
"mov ecx, 256",
"213:",
"mov eax, DWORD PTR [rdi]",
@@ -420,14 +456,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 213b",
- "mov eax, 2",
+ "mov eax, 4",
"mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
+ "mov eax, 5",
"mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
"add rdi, 1152",
"mov rsi, rbx",
- "add rsi, 16384",
+ "add rsi, 33792",
"mov rdx, rbx",
"add rdx, 2048",
"call {vg_mldsa_rej_ntt_poly}",
@@ -435,7 +471,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 16384",
+ "add rdi, 33792",
"mov ecx, 256",
"214:",
"mov eax, DWORD PTR [rdi]",
@@ -444,22 +480,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 214b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 17408",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 34816",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 17408",
+ "add rdi, 34816",
"mov ecx, 256",
"215:",
"mov eax, DWORD PTR [rdi]",
@@ -468,22 +503,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 215b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 18432",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 35840",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 18432",
+ "add rdi, 35840",
"mov ecx, 256",
"216:",
"mov eax, DWORD PTR [rdi]",
@@ -492,22 +526,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 216b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 19456",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 36864",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 19456",
+ "add rdi, 36864",
"mov ecx, 256",
"217:",
"mov eax, DWORD PTR [rdi]",
@@ -516,22 +549,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 217b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 20480",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 37888",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 20480",
+ "add rdi, 37888",
"mov ecx, 256",
"218:",
"mov eax, DWORD PTR [rdi]",
@@ -540,22 +572,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 218b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 21504",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 38912",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 21504",
+ "add rdi, 38912",
"mov ecx, 256",
"219:",
"mov eax, DWORD PTR [rdi]",
@@ -564,22 +595,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 219b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 22528",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 39936",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 22528",
+ "add rdi, 39936",
"mov ecx, 256",
"220:",
"mov eax, DWORD PTR [rdi]",
@@ -588,22 +618,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 220b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 23552",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 40960",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 23552",
+ "add rdi, 40960",
"mov ecx, 256",
"221:",
"mov eax, DWORD PTR [rdi]",
@@ -612,22 +641,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 221b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 24576",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 41984",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 24576",
+ "add rdi, 41984",
"mov ecx, 256",
"222:",
"mov eax, DWORD PTR [rdi]",
@@ -636,22 +664,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 222b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 8",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 25600",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 43008",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 25600",
+ "add rdi, 43008",
"mov ecx, 256",
"223:",
"mov eax, DWORD PTR [rdi]",
@@ -660,22 +687,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 223b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 9",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 26624",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 44032",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 26624",
+ "add rdi, 44032",
"mov ecx, 256",
"224:",
"mov eax, DWORD PTR [rdi]",
@@ -684,22 +710,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 224b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 10",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 27648",
+ "add rdi, 1216",
+ "mov esi, 4",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 45056",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 27648",
+ "add rdi, 45056",
"mov ecx, 256",
"225:",
"mov eax, DWORD PTR [rdi]",
@@ -708,489 +733,92 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 225b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
+ "mov rdi, r12",
+ "add rdi, 0",
"mov rsi, rbx",
- "add rsi, 28672",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 28672",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"226:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 226b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
+ "mov rdi, r13",
+ "add rdi, 0",
"mov rsi, rbx",
- "add rsi, 29696",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 29696",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"227:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 227b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
+ "mov rdi, r13",
+ "add rdi, 32",
"mov rsi, rbx",
- "add rsi, 30720",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 30720",
- "mov ecx, 256",
+ "add rsi, 1120",
+ "mov ecx, 32",
"228:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 228b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 31744",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 31744",
- "mov ecx, 256",
- "229:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 229b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 32768",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 32768",
- "mov ecx, 256",
- "230:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 230b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 33792",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 33792",
- "mov ecx, 256",
- "231:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 231b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 34816",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
"mov rdi, rbx",
"add rdi, 34816",
- "mov ecx, 256",
- "232:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 232b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
"mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 35840",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 128",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
"add rdi, 35840",
- "mov ecx, 256",
- "233:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 233b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
"mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 36864",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 256",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
"add rdi, 36864",
- "mov ecx, 256",
- "234:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 234b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
"mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 37888",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 384",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
"add rdi, 37888",
- "mov ecx, 256",
- "235:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 235b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
"mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 38912",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 512",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
"add rdi, 38912",
- "mov ecx, 256",
- "236:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 236b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
"mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 39936",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 640",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
"add rdi, 39936",
- "mov ecx, 256",
- "237:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 237b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1280], al",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 768",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 40960",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 40960",
- "mov ecx, 256",
- "238:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 238b",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 41984",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 41984",
- "mov ecx, 256",
- "239:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 239b",
- "mov eax, 8",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 43008",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 43008",
- "mov ecx, 256",
- "240:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 240b",
- "mov eax, 9",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 44032",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 44032",
- "mov ecx, 256",
- "241:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 241b",
- "mov eax, 10",
- "mov BYTE PTR [rbx+1280], al",
- "mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 4",
- "mov rdx, rbx",
- "add rdx, 45056",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 45056",
- "mov ecx, 256",
- "242:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 242b",
- "mov rdi, r12",
- "add rdi, 0",
- "mov rsi, rbx",
- "add rsi, 1024",
- "mov ecx, 32",
- "243:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 243b",
- "mov rdi, r13",
- "add rdi, 0",
- "mov rsi, rbx",
- "add rsi, 1024",
- "mov ecx, 32",
- "244:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 244b",
- "mov rdi, r13",
- "add rdi, 32",
- "mov rsi, rbx",
- "add rsi, 1120",
- "mov ecx, 32",
- "245:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 245b",
- "mov rdi, rbx",
- "add rdi, 34816",
- "mov esi, 4",
- "mov edx, 4",
- "mov rcx, r13",
- "add rcx, 128",
- "mov r8d, 128",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 35840",
- "mov esi, 4",
- "mov edx, 4",
- "mov rcx, r13",
- "add rcx, 256",
- "mov r8d, 128",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 36864",
- "mov esi, 4",
- "mov edx, 4",
- "mov rcx, r13",
- "add rcx, 384",
- "mov r8d, 128",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov esi, 4",
- "mov edx, 4",
- "mov rcx, r13",
- "add rcx, 512",
- "mov r8d, 128",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov esi, 4",
- "mov edx, 4",
- "mov rcx, r13",
- "add rcx, 640",
- "mov r8d, 128",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov esi, 4",
- "mov edx, 4",
- "mov rcx, r13",
- "add rcx, 768",
- "mov r8d, 128",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 40960",
+ "add rdi, 40960",
"mov esi, 4",
"mov edx, 4",
"mov rcx, r13",
@@ -1721,6 +1349,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32
vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
vg_keccak_pad = sym super::sha3::vg_keccak_pad,
vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2,
vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
@@ -4986,678 +4615,306 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"jne 21b",
"mov eax, 0",
"mov BYTE PTR [rbx+1281], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 4096",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 4096",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"22:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 22b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1442",
"mov rsi, rbx",
- "add rsi, 5120",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 5120",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"23:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 23b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1476",
"mov rsi, rbx",
- "add rsi, 6144",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"24:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 24b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 7168",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 7168",
- "mov ecx, 256",
- "25:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 25b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 8192",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 8192",
- "mov ecx, 256",
- "26:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 26b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 9216",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 9216",
- "mov ecx, 256",
- "27:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 27b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 10240",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 10240",
- "mov ecx, 256",
- "28:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 28b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 11264",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 11264",
- "mov ecx, 256",
- "29:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 29b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 12288",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 12288",
- "mov ecx, 256",
- "210:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 210b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 13312",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 13312",
- "mov ecx, 256",
- "211:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 211b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 14336",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 14336",
- "mov ecx, 256",
- "212:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 212b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 15360",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 15360",
- "mov ecx, 256",
- "213:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 213b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 16384",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 16384",
- "mov ecx, 256",
- "214:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 214b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 17408",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 17408",
- "mov ecx, 256",
- "215:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 215b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 18432",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 18432",
- "mov ecx, 256",
- "216:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 216b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 19456",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 19456",
- "mov ecx, 256",
- "217:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 217b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 20480",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 20480",
- "mov ecx, 256",
- "218:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 218b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 21504",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 21504",
- "mov ecx, 256",
- "219:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 219b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 22528",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 22528",
- "mov ecx, 256",
- "220:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
- "jne 220b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
+ "jne 24b",
+ "mov rdi, rbx",
+ "add rdi, 1510",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "25:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 23552",
+ "add rsi, 4096",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 23552",
- "mov ecx, 256",
- "221:",
+ "add rdi, 4096",
+ "mov ecx, 1024",
+ "26:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 221b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "jne 26b",
"mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 24576",
+ "add rsi, 8192",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 24576",
- "mov ecx, 256",
- "222:",
+ "add rdi, 8192",
+ "mov ecx, 1024",
+ "27:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 222b",
+ "jne 27b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 25600",
+ "add rsi, 12288",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 25600",
- "mov ecx, 256",
- "223:",
+ "add rdi, 12288",
+ "mov ecx, 1024",
+ "28:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 223b",
+ "jne 28b",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 26624",
+ "add rsi, 16384",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 26624",
- "mov ecx, 256",
- "224:",
+ "add rdi, 16384",
+ "mov ecx, 1024",
+ "29:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 224b",
+ "jne 29b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 27648",
+ "add rsi, 20480",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 27648",
- "mov ecx, 256",
- "225:",
+ "add rdi, 20480",
+ "mov ecx, 1024",
+ "210:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 225b",
+ "jne 210b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 28672",
+ "add rsi, 24576",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 28672",
- "mov ecx, 256",
- "226:",
+ "add rdi, 24576",
+ "mov ecx, 1024",
+ "211:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 226b",
+ "jne 211b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 29696",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 29696",
- "mov ecx, 256",
- "227:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 227b",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1508], al",
"mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 30720",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 30720",
- "mov ecx, 256",
- "228:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 228b",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1542], al",
"mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 31744",
+ "add rsi, 28672",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 49152",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 31744",
- "mov ecx, 256",
- "229:",
+ "add rdi, 28672",
+ "mov ecx, 1024",
+ "212:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 229b",
+ "jne 212b",
"mov eax, 3",
"mov BYTE PTR [rbx+1184], al",
"mov eax, 5",
@@ -5675,13 +4932,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 32768",
"mov ecx, 256",
- "230:",
+ "213:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 230b",
+ "jne 213b",
"mov eax, 4",
"mov BYTE PTR [rbx+1184], al",
"mov eax, 5",
@@ -5699,13 +4956,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 33792",
"mov ecx, 256",
- "231:",
+ "214:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 231b",
+ "jne 214b",
"mov eax, 0",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5722,13 +4979,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 34816",
"mov ecx, 256",
- "232:",
+ "215:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 232b",
+ "jne 215b",
"mov eax, 1",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5745,13 +5002,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 35840",
"mov ecx, 256",
- "233:",
+ "216:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 233b",
+ "jne 216b",
"mov eax, 2",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5768,13 +5025,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 36864",
"mov ecx, 256",
- "234:",
+ "217:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 234b",
+ "jne 217b",
"mov eax, 3",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5791,13 +5048,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 37888",
"mov ecx, 256",
- "235:",
+ "218:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 235b",
+ "jne 218b",
"mov eax, 4",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5814,13 +5071,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 38912",
"mov ecx, 256",
- "236:",
+ "219:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 236b",
+ "jne 219b",
"mov eax, 5",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5837,13 +5094,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 39936",
"mov ecx, 256",
- "237:",
+ "220:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 237b",
+ "jne 220b",
"mov eax, 6",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5860,13 +5117,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 40960",
"mov ecx, 256",
- "238:",
+ "221:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 238b",
+ "jne 221b",
"mov eax, 7",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5883,13 +5140,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 41984",
"mov ecx, 256",
- "239:",
+ "222:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 239b",
+ "jne 222b",
"mov eax, 8",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5906,13 +5163,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 43008",
"mov ecx, 256",
- "240:",
+ "223:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 240b",
+ "jne 223b",
"mov eax, 9",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5929,13 +5186,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 44032",
"mov ecx, 256",
- "241:",
+ "224:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 241b",
+ "jne 224b",
"mov eax, 10",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -5952,49 +5209,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 45056",
"mov ecx, 256",
- "242:",
+ "225:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 242b",
+ "jne 225b",
"mov rdi, r12",
"add rdi, 0",
"mov rsi, rbx",
"add rsi, 1024",
"mov ecx, 32",
- "243:",
+ "226:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 243b",
+ "jne 226b",
"mov rdi, r13",
"add rdi, 0",
"mov rsi, rbx",
"add rsi, 1024",
"mov ecx, 32",
- "244:",
+ "227:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 244b",
+ "jne 227b",
"mov rdi, r13",
"add rdi, 32",
"mov rsi, rbx",
"add rsi, 1120",
"mov ecx, 32",
- "245:",
+ "228:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 245b",
+ "jne 228b",
"mov rdi, rbx",
"add rdi, 34816",
"mov esi, 4",
@@ -6575,6 +5832,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen(seed: *const [u8; 32], pk
vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
vg_keccak_pad = sym super::sha3::vg_keccak_pad,
vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4,
vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs
index 50336abce..f8dfeb95d 100644
--- a/src/asm/x86_64/mldsa87.rs
+++ b/src/asm/x86_64/mldsa87.rs
@@ -132,119 +132,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"jne 21b",
"mov eax, 0",
"mov BYTE PTR [rbx+1281], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 4096",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 4096",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"22:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 22b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1442",
"mov rsi, rbx",
- "add rsi, 5120",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 5120",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"23:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 23b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1476",
"mov rsi, rbx",
- "add rsi, 6144",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"24:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 24b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1510",
"mov rsi, rbx",
- "add rsi, 7168",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 7168",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"25:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 25b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
"mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 8192",
+ "add rsi, 4096",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 8192",
- "mov ecx, 256",
+ "add rdi, 4096",
+ "mov ecx, 1024",
"26:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -252,23 +216,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 26b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 9216",
+ "add rsi, 8192",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 9216",
- "mov ecx, 256",
+ "add rdi, 8192",
+ "mov ecx, 1024",
"27:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -276,23 +252,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 27b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 10240",
+ "add rsi, 12288",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 10240",
- "mov ecx, 256",
+ "add rdi, 12288",
+ "mov ecx, 1024",
"28:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -300,23 +288,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 28b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 11264",
+ "add rsi, 16384",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 11264",
- "mov ecx, 256",
+ "add rdi, 16384",
+ "mov ecx, 1024",
"29:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -324,23 +324,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 29b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 12288",
+ "add rsi, 20480",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 12288",
- "mov ecx, 256",
+ "add rdi, 20480",
+ "mov ecx, 1024",
"210:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -348,23 +360,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 210b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 13312",
+ "add rsi, 24576",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 13312",
- "mov ecx, 256",
+ "add rdi, 24576",
+ "mov ecx, 1024",
"211:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -373,22 +397,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"sub rcx, 1",
"jne 211b",
"mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 14336",
+ "add rsi, 28672",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 14336",
- "mov ecx, 256",
+ "add rdi, 28672",
+ "mov ecx, 1024",
"212:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -396,23 +432,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 212b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1440], al",
"mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 15360",
+ "add rsi, 32768",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 15360",
- "mov ecx, 256",
+ "add rdi, 32768",
+ "mov ecx, 1024",
"213:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -420,23 +468,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 213b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 16384",
+ "add rsi, 36864",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 16384",
- "mov ecx, 256",
+ "add rdi, 36864",
+ "mov ecx, 1024",
"214:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -444,23 +504,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 214b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
"mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 17408",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1543], al",
+ "mov rdi, rbx",
+ "add rdi, 1408",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 17408",
- "mov ecx, 256",
+ "add rdi, 40960",
+ "mov ecx, 1024",
"215:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -468,23 +540,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 215b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 18432",
+ "add rsi, 45056",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 18432",
- "mov ecx, 256",
+ "add rdi, 45056",
+ "mov ecx, 1024",
"216:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -492,23 +576,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 216b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 19456",
+ "add rsi, 49152",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 19456",
- "mov ecx, 256",
+ "add rdi, 49152",
+ "mov ecx, 1024",
"217:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -516,23 +612,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 217b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 20480",
+ "add rsi, 53248",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 20480",
- "mov ecx, 256",
+ "add rdi, 53248",
+ "mov ecx, 1024",
"218:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -541,22 +649,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"sub rcx, 1",
"jne 218b",
"mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 21504",
+ "add rsi, 57344",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4_avx2}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 21504",
- "mov ecx, 256",
+ "add rdi, 57344",
+ "mov ecx, 1024",
"219:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
@@ -564,22 +684,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 219b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 22528",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 61440",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 22528",
+ "add rdi, 61440",
"mov ecx, 256",
"220:",
"mov eax, DWORD PTR [rdi]",
@@ -588,22 +707,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 220b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 23552",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 62464",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 23552",
+ "add rdi, 62464",
"mov ecx, 256",
"221:",
"mov eax, DWORD PTR [rdi]",
@@ -612,22 +730,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 221b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 24576",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 63488",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 24576",
+ "add rdi, 63488",
"mov ecx, 256",
"222:",
"mov eax, DWORD PTR [rdi]",
@@ -636,22 +753,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 222b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
"mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 25600",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 64512",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 25600",
+ "add rdi, 64512",
"mov ecx, 256",
"223:",
"mov eax, DWORD PTR [rdi]",
@@ -660,22 +776,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 223b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 26624",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 65536",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 26624",
+ "add rdi, 65536",
"mov ecx, 256",
"224:",
"mov eax, DWORD PTR [rdi]",
@@ -684,22 +799,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 224b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 27648",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 66560",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 27648",
+ "add rdi, 66560",
"mov ecx, 256",
"225:",
"mov eax, DWORD PTR [rdi]",
@@ -708,22 +822,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 225b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 28672",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 67584",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 28672",
+ "add rdi, 67584",
"mov ecx, 256",
"226:",
"mov eax, DWORD PTR [rdi]",
@@ -732,22 +845,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 226b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 29696",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 68608",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 29696",
+ "add rdi, 68608",
"mov ecx, 256",
"227:",
"mov eax, DWORD PTR [rdi]",
@@ -756,22 +868,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 227b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 8",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 30720",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 69632",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 30720",
+ "add rdi, 69632",
"mov ecx, 256",
"228:",
"mov eax, DWORD PTR [rdi]",
@@ -780,22 +891,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 228b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 9",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 31744",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 70656",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 31744",
+ "add rdi, 70656",
"mov ecx, 256",
"229:",
"mov eax, DWORD PTR [rdi]",
@@ -804,22 +914,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 229b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 10",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 32768",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 71680",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 32768",
+ "add rdi, 71680",
"mov ecx, 256",
"230:",
"mov eax, DWORD PTR [rdi]",
@@ -828,22 +937,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 230b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 11",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 33792",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 72704",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 33792",
+ "add rdi, 72704",
"mov ecx, 256",
"231:",
"mov eax, DWORD PTR [rdi]",
@@ -852,22 +960,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 231b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 12",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 34816",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 73728",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 34816",
+ "add rdi, 73728",
"mov ecx, 256",
"232:",
"mov eax, DWORD PTR [rdi]",
@@ -876,22 +983,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 232b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 13",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 35840",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 74752",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 35840",
+ "add rdi, 74752",
"mov ecx, 256",
"233:",
"mov eax, DWORD PTR [rdi]",
@@ -900,22 +1006,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 233b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
+ "mov eax, 14",
+ "mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 36864",
+ "add rdi, 1216",
+ "mov esi, 2",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 75776",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 36864",
+ "add rdi, 75776",
"mov ecx, 256",
"234:",
"mov eax, DWORD PTR [rdi]",
@@ -924,1140 +1029,648 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 4",
"sub rcx, 1",
"jne 234b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
+ "mov rdi, r12",
+ "add rdi, 0",
"mov rsi, rbx",
- "add rsi, 37888",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"235:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 235b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
+ "mov rdi, r13",
+ "add rdi, 0",
"mov rsi, rbx",
- "add rsi, 38912",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov ecx, 256",
+ "add rsi, 1024",
+ "mov ecx, 32",
"236:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 236b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
+ "mov rdi, r13",
+ "add rdi, 32",
"mov rsi, rbx",
- "add rsi, 39936",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov ecx, 256",
+ "add rsi, 1120",
+ "mov ecx, 32",
"237:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
"jne 237b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 40960",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 40960",
- "mov ecx, 256",
- "238:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 238b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 61440",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 128",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 41984",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdi, 62464",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 224",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
- "add rdi, 41984",
- "mov ecx, 256",
- "239:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 239b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 63488",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 320",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 43008",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdi, 64512",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 416",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
- "add rdi, 43008",
- "mov ecx, 256",
- "240:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 240b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 65536",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 512",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 44032",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdi, 66560",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 608",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
- "add rdi, 44032",
- "mov ecx, 256",
- "241:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 241b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 67584",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 704",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 68608",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 800",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 69632",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 896",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 70656",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 992",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 71680",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1088",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 72704",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1184",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 73728",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1280",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 74752",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1376",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 75776",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1472",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 61440",
"mov rsi, rbx",
- "add rsi, 45056",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 45056",
- "mov ecx, 256",
- "242:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 242b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 62464",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 63488",
"mov rsi, rbx",
- "add rsi, 46080",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 46080",
- "mov ecx, 256",
- "243:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 243b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 64512",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 65536",
"mov rsi, rbx",
- "add rsi, 47104",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 47104",
- "mov ecx, 256",
- "244:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 244b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 66560",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 67584",
"mov rsi, rbx",
- "add rsi, 48128",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 48128",
- "mov ecx, 256",
- "245:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 245b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 49152",
+ "add rsi, 5120",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 49152",
- "mov ecx, 256",
- "246:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 246b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 50176",
+ "add rsi, 7168",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 50176",
- "mov ecx, 256",
- "247:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 247b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 51200",
+ "add rsi, 9216",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 51200",
- "mov ecx, 256",
- "248:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 248b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 52224",
+ "add rsi, 10240",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 52224",
- "mov ecx, 256",
- "249:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 249b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 53248",
+ "add rsi, 68608",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
"mov rdi, rbx",
- "add rdi, 53248",
- "mov ecx, 256",
- "250:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 250b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 32",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 1568",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 54272",
+ "add rsi, 11264",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 54272",
- "mov ecx, 256",
- "251:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 251b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 12288",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 55296",
+ "add rsi, 13312",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 55296",
- "mov ecx, 256",
- "252:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 252b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 14336",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 56320",
+ "add rsi, 15360",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 56320",
- "mov ecx, 256",
- "253:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 253b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 16384",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 57344",
+ "add rsi, 17408",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 57344",
- "mov ecx, 256",
- "254:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 254b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 58368",
+ "add rsi, 69632",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
"mov rdi, rbx",
- "add rdi, 58368",
- "mov ecx, 256",
- "255:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 255b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 352",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 1984",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 59392",
+ "add rsi, 18432",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 59392",
- "mov ecx, 256",
- "256:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 256b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 19456",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 60416",
+ "add rsi, 20480",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 60416",
- "mov ecx, 256",
- "257:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 257b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 21504",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 22528",
"mov rdx, rbx",
- "add rdx, 61440",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 61440",
- "mov ecx, 256",
- "258:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 258b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 24576",
"mov rdx, rbx",
- "add rdx, 62464",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 62464",
- "mov ecx, 256",
- "259:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 259b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
"mov rdx, rbx",
- "add rdx, 63488",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
"mov rdi, rbx",
- "add rdi, 63488",
- "mov ecx, 256",
- "260:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 260b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 672",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 2400",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 25600",
"mov rdx, rbx",
- "add rdx, 64512",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 64512",
- "mov ecx, 256",
- "261:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 261b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 27648",
"mov rdx, rbx",
- "add rdx, 65536",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 65536",
- "mov ecx, 256",
- "262:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 262b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 29696",
"mov rdx, rbx",
- "add rdx, 66560",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 66560",
- "mov ecx, 256",
- "263:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 263b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 31744",
"mov rdx, rbx",
"add rdx, 67584",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 67584",
- "mov ecx, 256",
- "264:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 264b",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
"mov rdx, rbx",
- "add rdx, 68608",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
"mov rdi, rbx",
- "add rdi, 68608",
- "mov ecx, 256",
- "265:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 265b",
- "mov eax, 8",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 992",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 2816",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 32768",
"mov rdx, rbx",
- "add rdx, 69632",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 69632",
- "mov ecx, 256",
- "266:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 266b",
- "mov eax, 9",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 34816",
"mov rdx, rbx",
- "add rdx, 70656",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 70656",
- "mov ecx, 256",
- "267:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 267b",
- "mov eax, 10",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 35840",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 36864",
"mov rdx, rbx",
- "add rdx, 71680",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 71680",
- "mov ecx, 256",
- "268:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 268b",
- "mov eax, 11",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 38912",
"mov rdx, rbx",
- "add rdx, 72704",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 72704",
- "mov ecx, 256",
- "269:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 269b",
- "mov eax, 12",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
"mov rdx, rbx",
- "add rdx, 73728",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
"mov rdi, rbx",
- "add rdi, 73728",
- "mov ecx, 256",
- "270:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 270b",
- "mov eax, 13",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 1312",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 3232",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 39936",
"mov rdx, rbx",
- "add rdx, 74752",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 74752",
- "mov ecx, 256",
- "271:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 271b",
- "mov eax, 14",
- "mov BYTE PTR [rbx+1280], al",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 1216",
- "mov esi, 2",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 41984",
"mov rdx, rbx",
- "add rdx, 75776",
- "mov rcx, rbx",
- "add rcx, 2048",
- "call {vg_mldsa_rej_bounded_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 75776",
- "mov ecx, 256",
- "272:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 272b",
- "mov rdi, r12",
- "add rdi, 0",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 1024",
- "mov ecx, 32",
- "273:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 273b",
- "mov rdi, r13",
- "add rdi, 0",
+ "add rsi, 43008",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 1024",
- "mov ecx, 32",
- "274:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 274b",
- "mov rdi, r13",
- "add rdi, 32",
+ "add rsi, 44032",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 1120",
- "mov ecx, 32",
- "275:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 275b",
- "mov rdi, rbx",
- "add rdi, 61440",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 128",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 62464",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 224",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 63488",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 320",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 64512",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 416",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 65536",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 512",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 66560",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 608",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 67584",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 704",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 68608",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 800",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 69632",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 896",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 70656",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 992",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 71680",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 1088",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 72704",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 1184",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 73728",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 1280",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 74752",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 1376",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 75776",
- "mov esi, 2",
- "mov edx, 2",
- "mov rcx, r13",
- "add rcx, 1472",
- "mov r8d, 96",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 61440",
- "mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 62464",
- "mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 63488",
- "mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 64512",
- "mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 65536",
- "mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 66560",
- "mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 67584",
- "mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 4096",
- "mov rdx, rbx",
- "add rdx, 61440",
- "call {vg_mldsa_multiply_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 5120",
- "mov rdx, rbx",
- "add rdx, 62464",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 6144",
- "mov rdx, rbx",
- "add rdx, 63488",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 7168",
- "mov rdx, rbx",
- "add rdx, 64512",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 8192",
- "mov rdx, rbx",
- "add rdx, 65536",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 9216",
- "mov rdx, rbx",
- "add rdx, 66560",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 10240",
+ "add rsi, 46080",
"mov rdx, rbx",
"add rdx, 67584",
"call {vg_mldsa_multiply_add_ntt_avx2}",
@@ -2069,7 +1682,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 68608",
+ "add rsi, 73728",
"call {vg_mldsa_add_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
@@ -2082,7 +1695,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 77824",
"mov esi, 1023",
"mov rdx, r12",
- "add rdx, 32",
+ "add rdx, 1632",
"mov ecx, 320",
"call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
@@ -2090,55 +1703,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"mov esi, 4095",
"mov edx, 4096",
"mov rcx, r13",
- "add rcx, 1568",
+ "add rcx, 3648",
"mov r8d, 416",
"call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 11264",
+ "add rsi, 47104",
"mov rdx, rbx",
"add rdx, 61440",
"call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 12288",
+ "add rsi, 48128",
"mov rdx, rbx",
"add rdx, 62464",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 13312",
+ "add rsi, 49152",
"mov rdx, rbx",
"add rdx, 63488",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 14336",
+ "add rsi, 50176",
"mov rdx, rbx",
"add rdx, 64512",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 15360",
+ "add rsi, 51200",
"mov rdx, rbx",
"add rdx, 65536",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 16384",
+ "add rsi, 52224",
"mov rdx, rbx",
"add rdx, 66560",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 17408",
+ "add rsi, 53248",
"mov rdx, rbx",
"add rdx, 67584",
"call {vg_mldsa_multiply_add_ntt_avx2}",
@@ -2150,7 +1763,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 69632",
+ "add rsi, 74752",
"call {vg_mldsa_add_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
@@ -2163,7 +1776,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 77824",
"mov esi, 1023",
"mov rdx, r12",
- "add rdx, 352",
+ "add rdx, 1952",
"mov ecx, 320",
"call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
@@ -2171,55 +1784,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"mov esi, 4095",
"mov edx, 4096",
"mov rcx, r13",
- "add rcx, 1984",
+ "add rcx, 4064",
"mov r8d, 416",
"call {vg_mldsa_bit_pack}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 18432",
+ "add rsi, 54272",
"mov rdx, rbx",
"add rdx, 61440",
"call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 19456",
+ "add rsi, 55296",
"mov rdx, rbx",
"add rdx, 62464",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 20480",
+ "add rsi, 56320",
"mov rdx, rbx",
"add rdx, 63488",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 21504",
+ "add rsi, 57344",
"mov rdx, rbx",
"add rdx, 64512",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 22528",
+ "add rsi, 58368",
"mov rdx, rbx",
"add rdx, 65536",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 23552",
+ "add rsi, 59392",
"mov rdx, rbx",
"add rdx, 66560",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 24576",
+ "add rsi, 60416",
"mov rdx, rbx",
"add rdx, 67584",
"call {vg_mldsa_multiply_add_ntt_avx2}",
@@ -2231,7 +1844,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"mov rdi, rbx",
"add rdi, 76800",
"mov rsi, rbx",
- "add rsi, 70656",
+ "add rsi, 75776",
"call {vg_mldsa_add_avx2}",
"mov rdi, rbx",
"add rdi, 76800",
@@ -2244,7 +1857,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"add rdi, 77824",
"mov esi, 1023",
"mov rdx, r12",
- "add rdx, 672",
+ "add rdx, 2272",
"mov ecx, 320",
"call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
@@ -2252,2297 +1865,1892 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32
"mov esi, 4095",
"mov edx, 4096",
"mov rcx, r13",
- "add rcx, 2400",
+ "add rcx, 4480",
"mov r8d, 416",
"call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 25600",
- "mov rdx, rbx",
- "add rdx, 61440",
- "call {vg_mldsa_multiply_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 26624",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 2592",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 8",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r13",
+ "add rcx, 64",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2,
+ vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
+ vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2,
+ vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ )
+}
+
+/// The CPU features `vg_mldsa87_sign_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA87_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-87 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
+///
+/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness.
+///
+/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address.
+///
+/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing.
+///
+/// # Safety
+///
+/// * `sk` must be valid for reads of 4896 bytes.
+/// * `mu` must be valid for reads of 64 bytes.
+/// * `rnd` must be valid for reads of 32 bytes.
+/// * `sig` must be valid for reads and writes of 4627 bytes.
+/// * `scratch` must be valid for reads and writes of 144384 bytes.
+/// * `sk` must have been written by `vg_mldsa87_keygen`.
+/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do).
+/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 4627], scratch: *mut [u64; 18048]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [r8+840], rbx",
+ "mov QWORD PTR [r8+848], rbp",
+ "mov QWORD PTR [r8+856], r12",
+ "mov QWORD PTR [r8+864], r13",
+ "mov QWORD PTR [r8+872], r14",
+ "mov QWORD PTR [r8+880], r15",
+ "mov rbx, r8",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r14, rcx",
+ "mov r15d, 1",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbp",
+ "add rsi, 0",
+ "mov ecx, 32",
+ "20:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 20b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 64512",
"mov rdx, rbx",
- "add rdx, 62464",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 27648",
+ "add rsi, 65536",
"mov rdx, rbx",
- "add rdx, 63488",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 28672",
+ "add rsi, 66560",
"mov rdx, rbx",
- "add rdx, 64512",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 29696",
+ "add rsi, 67584",
"mov rdx, rbx",
- "add rdx, 65536",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 30720",
+ "add rsi, 68608",
"mov rdx, rbx",
- "add rdx, 66560",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 31744",
+ "add rsi, 69632",
"mov rdx, rbx",
- "add rdx, 67584",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_inv_ntt_avx2}",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
"add rsi, 71680",
- "call {vg_mldsa_add_avx2}",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 77824",
+ "add rsi, 72704",
"mov rdx, rbx",
- "add rdx, 78848",
- "call {vg_mldsa_power2round}",
- "mov rdi, rbx",
- "add rdi, 77824",
- "mov esi, 1023",
- "mov rdx, r12",
- "add rdx, 992",
- "mov ecx, 320",
- "call {vg_mldsa_simple_bit_pack}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 78848",
- "mov esi, 4095",
- "mov edx, 4096",
- "mov rcx, r13",
- "add rcx, 2816",
- "mov r8d, 416",
- "call {vg_mldsa_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 32768",
+ "add rsi, 73728",
"mov rdx, rbx",
- "add rdx, 61440",
- "call {vg_mldsa_multiply_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 33792",
+ "add rsi, 74752",
"mov rdx, rbx",
- "add rdx, 62464",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 34816",
+ "add rsi, 75776",
"mov rdx, rbx",
- "add rdx, 63488",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 35840",
+ "add rsi, 76800",
"mov rdx, rbx",
- "add rdx, 64512",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 36864",
+ "add rsi, 77824",
"mov rdx, rbx",
- "add rdx, 65536",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 37888",
+ "add rsi, 78848",
"mov rdx, rbx",
- "add rdx, 66560",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 38912",
+ "add rsi, 79872",
"mov rdx, rbx",
- "add rdx, 67584",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_inv_ntt_avx2}",
+ "add rsi, 80896",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 72704",
- "call {vg_mldsa_add_avx2}",
+ "add rsi, 81920",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 77824",
+ "add rsi, 82944",
"mov rdx, rbx",
- "add rdx, 78848",
- "call {vg_mldsa_power2round}",
- "mov rdi, rbx",
- "add rdi, 77824",
- "mov esi, 1023",
- "mov rdx, r12",
- "add rdx, 1312",
- "mov ecx, 320",
- "call {vg_mldsa_simple_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 78848",
- "mov esi, 4095",
- "mov edx, 4096",
- "mov rcx, r13",
- "add rcx, 3232",
- "mov r8d, 416",
- "call {vg_mldsa_bit_pack}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 39936",
+ "add rsi, 83968",
"mov rdx, rbx",
- "add rdx, 61440",
- "call {vg_mldsa_multiply_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 40960",
+ "add rsi, 84992",
"mov rdx, rbx",
- "add rdx, 62464",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 41984",
+ "add rsi, 86016",
"mov rdx, rbx",
- "add rdx, 63488",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 43008",
+ "add rsi, 87040",
"mov rdx, rbx",
- "add rdx, 64512",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 44032",
+ "add rsi, 88064",
"mov rdx, rbx",
- "add rdx, 65536",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 45056",
+ "add rsi, 89088",
"mov rdx, rbx",
- "add rdx, 66560",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 46080",
+ "add rsi, 90112",
"mov rdx, rbx",
- "add rdx, 67584",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_inv_ntt_avx2}",
+ "add rsi, 91136",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 73728",
- "call {vg_mldsa_add_avx2}",
+ "add rsi, 92160",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 77824",
+ "add rsi, 93184",
"mov rdx, rbx",
- "add rdx, 78848",
- "call {vg_mldsa_power2round}",
- "mov rdi, rbx",
- "add rdi, 77824",
- "mov esi, 1023",
- "mov rdx, r12",
- "add rdx, 1632",
- "mov ecx, 320",
- "call {vg_mldsa_simple_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 78848",
- "mov esi, 4095",
- "mov edx, 4096",
- "mov rcx, r13",
- "add rcx, 3648",
- "mov r8d, 416",
- "call {vg_mldsa_bit_pack}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 47104",
+ "add rsi, 94208",
"mov rdx, rbx",
- "add rdx, 61440",
- "call {vg_mldsa_multiply_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 48128",
+ "add rsi, 95232",
"mov rdx, rbx",
- "add rdx, 62464",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 49152",
+ "add rsi, 96256",
"mov rdx, rbx",
- "add rdx, 63488",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 50176",
+ "add rsi, 97280",
"mov rdx, rbx",
- "add rdx, 64512",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 51200",
+ "add rsi, 98304",
"mov rdx, rbx",
- "add rdx, 65536",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 52224",
+ "add rsi, 99328",
"mov rdx, rbx",
- "add rdx, 66560",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 53248",
+ "add rsi, 100352",
"mov rdx, rbx",
- "add rdx, 67584",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_inv_ntt_avx2}",
+ "add rsi, 101376",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 74752",
- "call {vg_mldsa_add_avx2}",
+ "add rsi, 102400",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 77824",
+ "add rsi, 103424",
"mov rdx, rbx",
- "add rdx, 78848",
- "call {vg_mldsa_power2round}",
- "mov rdi, rbx",
- "add rdi, 77824",
- "mov esi, 1023",
- "mov rdx, r12",
- "add rdx, 1952",
- "mov ecx, 320",
- "call {vg_mldsa_simple_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 78848",
- "mov esi, 4095",
- "mov edx, 4096",
- "mov rcx, r13",
- "add rcx, 4064",
- "mov r8d, 416",
- "call {vg_mldsa_bit_pack}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 54272",
+ "add rsi, 104448",
"mov rdx, rbx",
- "add rdx, 61440",
- "call {vg_mldsa_multiply_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 55296",
+ "add rsi, 105472",
"mov rdx, rbx",
- "add rdx, 62464",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 56320",
+ "add rsi, 106496",
"mov rdx, rbx",
- "add rdx, 63488",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 57344",
+ "add rsi, 107520",
"mov rdx, rbx",
- "add rdx, 64512",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 58368",
+ "add rsi, 108544",
"mov rdx, rbx",
- "add rdx, 65536",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
- "add rdi, 76800",
+ "add rdi, 912",
"mov rsi, rbx",
- "add rsi, 59392",
- "mov rdx, rbx",
- "add rdx, 66560",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 60416",
- "mov rdx, rbx",
- "add rdx, 67584",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 2048",
- "call {vg_mldsa_inv_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 75776",
- "call {vg_mldsa_add_avx2}",
- "mov rdi, rbx",
- "add rdi, 76800",
- "mov rsi, rbx",
- "add rsi, 77824",
- "mov rdx, rbx",
- "add rdx, 78848",
- "call {vg_mldsa_power2round}",
- "mov rdi, rbx",
- "add rdi, 77824",
- "mov esi, 1023",
- "mov rdx, r12",
- "add rdx, 2272",
- "mov ecx, 320",
- "call {vg_mldsa_simple_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 78848",
- "mov esi, 4095",
- "mov edx, 4096",
- "mov rcx, r13",
- "add rcx, 4480",
- "mov r8d, 416",
- "call {vg_mldsa_bit_pack}",
- "mov eax, 0",
- "mov QWORD PTR [rbx], rax",
- "mov QWORD PTR [rbx+8], rax",
- "mov QWORD PTR [rbx+16], rax",
- "mov QWORD PTR [rbx+24], rax",
- "mov QWORD PTR [rbx+32], rax",
- "mov QWORD PTR [rbx+40], rax",
- "mov QWORD PTR [rbx+48], rax",
- "mov QWORD PTR [rbx+56], rax",
- "mov QWORD PTR [rbx+64], rax",
- "mov QWORD PTR [rbx+72], rax",
- "mov QWORD PTR [rbx+80], rax",
- "mov QWORD PTR [rbx+88], rax",
- "mov QWORD PTR [rbx+96], rax",
- "mov QWORD PTR [rbx+104], rax",
- "mov QWORD PTR [rbx+112], rax",
- "mov QWORD PTR [rbx+120], rax",
- "mov QWORD PTR [rbx+128], rax",
- "mov QWORD PTR [rbx+136], rax",
- "mov QWORD PTR [rbx+144], rax",
- "mov QWORD PTR [rbx+152], rax",
- "mov QWORD PTR [rbx+160], rax",
- "mov QWORD PTR [rbx+168], rax",
- "mov QWORD PTR [rbx+176], rax",
- "mov QWORD PTR [rbx+184], rax",
- "mov QWORD PTR [rbx+192], rax",
- "mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 0",
- "mov rcx, r12",
- "add rcx, 0",
- "mov r8d, 2592",
- "mov r9, rbx",
- "add r9, 200",
- "call {vg_keccak_absorb}",
- "mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 8",
- "mov ecx, 31",
- "mov r8, rbx",
- "add r8, 200",
- "call {vg_keccak_pad}",
- "mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 0",
- "mov rcx, r13",
- "add rcx, 64",
- "mov r8d, 64",
- "mov r9, rbx",
- "add r9, 200",
- "call {vg_keccak_squeeze}",
- "mov eax, r15d",
- "mov r15, QWORD PTR [rbx+880]",
- "mov r14, QWORD PTR [rbx+872]",
- "mov r13, QWORD PTR [rbx+864]",
- "mov r12, QWORD PTR [rbx+856]",
- "mov rbp, QWORD PTR [rbx+848]",
- "mov rbx, QWORD PTR [rbx+840]",
- "ret",
- vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
- vg_keccak_pad = sym super::sha3::vg_keccak_pad,
- vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
- vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
- vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
- vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
- vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
- vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
- vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
- vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
- vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2,
- vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round,
- vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
- )
-}
-
-/// The CPU features `vg_mldsa87_sign_avx2` requires (`Artifact.features`).
-pub(crate) const VG_MLDSA87_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
-
-/// ML-DSA-87 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
-///
-/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness.
-///
-/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address.
-///
-/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing.
-///
-/// # Safety
-///
-/// * `sk` must be valid for reads of 4896 bytes.
-/// * `mu` must be valid for reads of 64 bytes.
-/// * `rnd` must be valid for reads of 32 bytes.
-/// * `sig` must be valid for reads and writes of 4627 bytes.
-/// * `scratch` must be valid for reads and writes of 144384 bytes.
-/// * `sk` must have been written by `vg_mldsa87_keygen`.
-/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing.
-/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
-/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do).
-/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
-/// * The CPU must support the `avx` and `avx2` target features.
-#[unsafe(naked)]
-pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 4627], scratch: *mut [u64; 18048]) -> u32 {
- core::arch::naked_asm!(
- "mov QWORD PTR [r8+840], rbx",
- "mov QWORD PTR [r8+848], rbp",
- "mov QWORD PTR [r8+856], r12",
- "mov QWORD PTR [r8+864], r13",
- "mov QWORD PTR [r8+872], r14",
- "mov QWORD PTR [r8+880], r15",
- "mov rbx, r8",
- "mov rbp, rdi",
- "mov r12, rsi",
- "mov r13, rdx",
- "mov r14, rcx",
- "mov r15d, 1",
- "mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbp",
- "add rsi, 0",
- "mov ecx, 32",
- "20:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 20b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+945], al",
- "mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 64512",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 1",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+945], al",
- "mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 65536",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 2",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+945], al",
- "mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 66560",
+ "add rsi, 109568",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 3",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 0",
+ "mov eax, 6",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 67584",
+ "add rsi, 110592",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 4",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 0",
+ "mov eax, 6",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 68608",
+ "add rsi, 111616",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 5",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 0",
+ "mov eax, 6",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 69632",
+ "add rsi, 112640",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 6",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 0",
+ "mov eax, 6",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 70656",
+ "add rsi, 113664",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 0",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 1",
+ "mov eax, 7",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 71680",
+ "add rsi, 114688",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 1",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 1",
+ "mov eax, 7",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 72704",
+ "add rsi, 115712",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 2",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 1",
+ "mov eax, 7",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 73728",
+ "add rsi, 116736",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 3",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 1",
+ "mov eax, 7",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 74752",
+ "add rsi, 117760",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 4",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 1",
+ "mov eax, 7",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 75776",
+ "add rsi, 118784",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 5",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 1",
+ "mov eax, 7",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 76800",
+ "add rsi, 119808",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
"mov eax, 6",
"mov BYTE PTR [rbx+944], al",
- "mov eax, 1",
+ "mov eax, 7",
"mov BYTE PTR [rbx+945], al",
"mov rdi, rbx",
"add rdi, 912",
"mov rsi, rbx",
- "add rsi, 77824",
+ "add rsi, 120832",
"mov rdx, rbx",
"add rdx, 3072",
"call {vg_mldsa_rej_ntt_poly}",
"and r15d, eax",
- "mov eax, 0",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+945], al",
- "mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 78848",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 1",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+945], al",
+ "test r15d, r15d",
+ "jne 21f",
+ "jmp 22f",
+ "21:",
+ "mov rdi, rbp",
+ "add rdi, 128",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 40960",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 40960",
"mov rsi, rbx",
- "add rsi, 79872",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 2",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 224",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 41984",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 41984",
"mov rsi, rbx",
- "add rsi, 80896",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 3",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 320",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 43008",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 43008",
"mov rsi, rbx",
- "add rsi, 81920",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 4",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 416",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 44032",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 44032",
"mov rsi, rbx",
- "add rsi, 82944",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 5",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 512",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 45056",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 45056",
"mov rsi, rbx",
- "add rsi, 83968",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 6",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 608",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 46080",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 46080",
"mov rsi, rbx",
- "add rsi, 84992",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 0",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 704",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 47104",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 47104",
"mov rsi, rbx",
- "add rsi, 86016",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 1",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 800",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 48128",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 48128",
"mov rsi, rbx",
- "add rsi, 87040",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 2",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 896",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 49152",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 49152",
"mov rsi, rbx",
- "add rsi, 88064",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 3",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 992",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 50176",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 50176",
"mov rsi, rbx",
- "add rsi, 89088",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 4",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1088",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 51200",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 51200",
"mov rsi, rbx",
- "add rsi, 90112",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 5",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1184",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 52224",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 52224",
"mov rsi, rbx",
- "add rsi, 91136",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 6",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+945], al",
- "mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 92160",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 0",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+945], al",
- "mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 93184",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 1",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1280",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 53248",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 53248",
"mov rsi, rbx",
- "add rsi, 94208",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 2",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1376",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 54272",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 54272",
"mov rsi, rbx",
- "add rsi, 95232",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 3",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1472",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 55296",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 55296",
"mov rsi, rbx",
- "add rsi, 96256",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 4",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1568",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 56320",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 56320",
"mov rsi, rbx",
- "add rsi, 97280",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 5",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1984",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 57344",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 57344",
"mov rsi, rbx",
- "add rsi, 98304",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 6",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 2400",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 58368",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 58368",
"mov rsi, rbx",
- "add rsi, 99328",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 0",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 2816",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 59392",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 59392",
"mov rsi, rbx",
- "add rsi, 100352",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 1",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 3232",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 60416",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 60416",
"mov rsi, rbx",
- "add rsi, 101376",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 2",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 3648",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 61440",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 61440",
"mov rsi, rbx",
- "add rsi, 102400",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 3",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 4064",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 62464",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 62464",
"mov rsi, rbx",
- "add rsi, 103424",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 4",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 4480",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 63488",
+ "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 63488",
"mov rsi, rbx",
- "add rsi, 104448",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 5",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
"mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 105472",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 6",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+945], al",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbp",
+ "add rcx, 32",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
"mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 106496",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 32",
+ "mov rcx, r13",
+ "add rcx, 0",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 128",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 960",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
"mov eax, 0",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+945], al",
+ "mov QWORD PTR [rbx+896], rax",
+ "mov eax, 814",
+ "mov QWORD PTR [rbx+888], rax",
+ "23:",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 0",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
"mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 107520",
+ "add rdi, 960",
+ "mov esi, 524288",
"mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 1",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+945], al",
+ "add rdx, 18432",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 25600",
"mov rsi, rbx",
- "add rsi, 108544",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 2",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 18432",
+ "mov ecx, 1024",
+ "24:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 24b",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 25600",
"mov rsi, rbx",
- "add rsi, 109568",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 3",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 1",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
"mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 110592",
+ "add rdi, 960",
+ "mov esi, 524288",
"mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 4",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+945], al",
+ "add rdx, 19456",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 26624",
"mov rsi, rbx",
- "add rsi, 111616",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 5",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 19456",
+ "mov ecx, 1024",
+ "25:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 25b",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 26624",
"mov rsi, rbx",
- "add rsi, 112640",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 6",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 2",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
"mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 113664",
+ "add rdi, 960",
+ "mov esi, 524288",
"mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 0",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+945], al",
+ "add rdx, 20480",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 27648",
"mov rsi, rbx",
- "add rsi, 114688",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 1",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 20480",
+ "mov ecx, 1024",
+ "26:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 26b",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 27648",
"mov rsi, rbx",
- "add rsi, 115712",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 2",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 3",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
"mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 116736",
+ "add rdi, 960",
+ "mov esi, 524288",
"mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 3",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+945], al",
+ "add rdx, 21504",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 28672",
"mov rsi, rbx",
- "add rsi, 117760",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 4",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 21504",
+ "mov ecx, 1024",
+ "27:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 27b",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 28672",
"mov rsi, rbx",
- "add rsi, 118784",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 5",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+945], al",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 4",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
"mov rdi, rbx",
- "add rdi, 912",
- "mov rsi, rbx",
- "add rsi, 119808",
+ "add rdi, 960",
+ "mov esi, 524288",
"mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov eax, 6",
- "mov BYTE PTR [rbx+944], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+945], al",
+ "add rdx, 22528",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
"mov rdi, rbx",
- "add rdi, 912",
+ "add rdi, 29696",
"mov rsi, rbx",
- "add rsi, 120832",
- "mov rdx, rbx",
- "add rdx, 3072",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "test r15d, r15d",
- "jne 21f",
- "jmp 22f",
- "21:",
- "mov rdi, rbp",
- "add rdi, 128",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 40960",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 22528",
+ "mov ecx, 1024",
+ "28:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 28b",
"mov rdi, rbx",
- "add rdi, 40960",
+ "add rdi, 29696",
"mov rsi, rbx",
"add rsi, 3072",
"call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 224",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 41984",
- "call {vg_mldsa_bit_unpack}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 5",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
"mov rdi, rbx",
- "add rdi, 41984",
- "mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 320",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 43008",
- "call {vg_mldsa_bit_unpack}",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
"mov rdi, rbx",
- "add rdi, 43008",
+ "add rdi, 30720",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 416",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 44032",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 23552",
+ "mov ecx, 1024",
+ "29:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 29b",
"mov rdi, rbx",
- "add rdi, 44032",
+ "add rdi, 30720",
"mov rsi, rbx",
"add rsi, 3072",
"call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 512",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 45056",
- "call {vg_mldsa_bit_unpack}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 6",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
"mov rdi, rbx",
- "add rdi, 45056",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 608",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 46080",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 24576",
+ "mov ecx, 1024",
+ "210:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 210b",
"mov rdi, rbx",
- "add rdi, 46080",
+ "add rdi, 31744",
"mov rsi, rbx",
"add rsi, 3072",
"call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 704",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 47104",
- "call {vg_mldsa_bit_unpack}",
"mov rdi, rbx",
- "add rdi, 47104",
+ "add rdi, 32768",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 800",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 48128",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 64512",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 48128",
+ "add rdi, 32768",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 896",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 49152",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 65536",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 49152",
+ "add rdi, 32768",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 992",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 50176",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 66560",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 50176",
+ "add rdi, 32768",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 1088",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 51200",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 67584",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 51200",
+ "add rdi, 32768",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 1184",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 52224",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 68608",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 52224",
+ "add rdi, 32768",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 1280",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 53248",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 69632",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 53248",
+ "add rdi, 32768",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 1376",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 54272",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 54272",
+ "add rdi, 32768",
"mov rsi, rbx",
"add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 1472",
- "mov esi, 96",
- "mov edx, 2",
- "mov ecx, 2",
- "mov r8, rbx",
- "add r8, 55296",
- "call {vg_mldsa_bit_unpack}",
+ "call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 55296",
+ "add rdi, 33792",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 1568",
- "mov esi, 416",
- "mov edx, 4095",
- "mov ecx, 4096",
- "mov r8, rbx",
- "add r8, 56320",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 71680",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 56320",
+ "add rdi, 33792",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 1984",
- "mov esi, 416",
- "mov edx, 4095",
- "mov ecx, 4096",
- "mov r8, rbx",
- "add r8, 57344",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 72704",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 57344",
+ "add rdi, 33792",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 2400",
- "mov esi, 416",
- "mov edx, 4095",
- "mov ecx, 4096",
- "mov r8, rbx",
- "add r8, 58368",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 73728",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 58368",
+ "add rdi, 33792",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 2816",
- "mov esi, 416",
- "mov edx, 4095",
- "mov ecx, 4096",
- "mov r8, rbx",
- "add r8, 59392",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 74752",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 59392",
+ "add rdi, 33792",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 3232",
- "mov esi, 416",
- "mov edx, 4095",
- "mov ecx, 4096",
- "mov r8, rbx",
- "add r8, 60416",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 75776",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 60416",
+ "add rdi, 33792",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 3648",
- "mov esi, 416",
- "mov edx, 4095",
- "mov ecx, 4096",
- "mov r8, rbx",
- "add r8, 61440",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 76800",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 61440",
+ "add rdi, 33792",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 4064",
- "mov esi, 416",
- "mov edx, 4095",
- "mov ecx, 4096",
- "mov r8, rbx",
- "add r8, 62464",
- "call {vg_mldsa_bit_unpack}",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 62464",
+ "add rdi, 33792",
"mov rsi, rbx",
"add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbp",
- "add rdi, 4480",
- "mov esi, 416",
- "mov edx, 4095",
- "mov ecx, 4096",
- "mov r8, rbx",
- "add r8, 63488",
- "call {vg_mldsa_bit_unpack}",
+ "call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 63488",
+ "add rdi, 34816",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov eax, 0",
- "mov QWORD PTR [rbx], rax",
- "mov QWORD PTR [rbx+8], rax",
- "mov QWORD PTR [rbx+16], rax",
- "mov QWORD PTR [rbx+24], rax",
- "mov QWORD PTR [rbx+32], rax",
- "mov QWORD PTR [rbx+40], rax",
- "mov QWORD PTR [rbx+48], rax",
- "mov QWORD PTR [rbx+56], rax",
- "mov QWORD PTR [rbx+64], rax",
- "mov QWORD PTR [rbx+72], rax",
- "mov QWORD PTR [rbx+80], rax",
- "mov QWORD PTR [rbx+88], rax",
- "mov QWORD PTR [rbx+96], rax",
- "mov QWORD PTR [rbx+104], rax",
- "mov QWORD PTR [rbx+112], rax",
- "mov QWORD PTR [rbx+120], rax",
- "mov QWORD PTR [rbx+128], rax",
- "mov QWORD PTR [rbx+136], rax",
- "mov QWORD PTR [rbx+144], rax",
- "mov QWORD PTR [rbx+152], rax",
- "mov QWORD PTR [rbx+160], rax",
- "mov QWORD PTR [rbx+168], rax",
- "mov QWORD PTR [rbx+176], rax",
- "mov QWORD PTR [rbx+184], rax",
- "mov QWORD PTR [rbx+192], rax",
- "mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 0",
- "mov rcx, rbp",
- "add rcx, 32",
- "mov r8d, 32",
- "mov r9, rbx",
- "add r9, 200",
- "call {vg_keccak_absorb}",
+ "add rsi, 78848",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 32",
- "mov rcx, r13",
- "add rcx, 0",
- "mov r8d, 32",
- "mov r9, rbx",
- "add r9, 200",
- "call {vg_keccak_absorb}",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 79872",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 64",
- "mov rcx, r12",
- "add rcx, 0",
- "mov r8d, 64",
- "mov r9, rbx",
- "add r9, 200",
- "call {vg_keccak_absorb}",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 80896",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 128",
- "mov ecx, 31",
- "mov r8, rbx",
- "add r8, 200",
- "call {vg_keccak_pad}",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 81920",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 0",
- "mov rcx, rbx",
- "add rcx, 960",
- "mov r8d, 64",
- "mov r9, rbx",
- "add r9, 200",
- "call {vg_keccak_squeeze}",
- "mov eax, 0",
- "mov QWORD PTR [rbx+896], rax",
- "mov eax, 814",
- "mov QWORD PTR [rbx+888], rax",
- "23:",
- "mov rax, QWORD PTR [rbx+896]",
- "add rax, 0",
- "mov BYTE PTR [rbx+1024], al",
- "shr rax, 8",
- "mov BYTE PTR [rbx+1025], al",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 82944",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 960",
- "mov esi, 524288",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 83968",
"mov rdx, rbx",
- "add rdx, 18432",
- "mov rcx, rbx",
- "add rcx, 3072",
- "call {vg_mldsa_expand_mask_poly}",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 25600",
+ "add rdi, 34816",
"mov rsi, rbx",
- "add rsi, 18432",
- "mov ecx, 1024",
- "24:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 24b",
+ "add rsi, 84992",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 25600",
+ "add rdi, 34816",
"mov rsi, rbx",
"add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rax, QWORD PTR [rbx+896]",
- "add rax, 1",
- "mov BYTE PTR [rbx+1024], al",
- "shr rax, 8",
- "mov BYTE PTR [rbx+1025], al",
+ "call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 960",
- "mov esi, 524288",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 86016",
"mov rdx, rbx",
- "add rdx, 19456",
- "mov rcx, rbx",
- "add rcx, 3072",
- "call {vg_mldsa_expand_mask_poly}",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 26624",
+ "add rdi, 35840",
"mov rsi, rbx",
- "add rsi, 19456",
- "mov ecx, 1024",
- "25:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 25b",
+ "add rsi, 87040",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 26624",
+ "add rdi, 35840",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rax, QWORD PTR [rbx+896]",
- "add rax, 2",
- "mov BYTE PTR [rbx+1024], al",
- "shr rax, 8",
- "mov BYTE PTR [rbx+1025], al",
- "mov rdi, rbx",
- "add rdi, 960",
- "mov esi, 524288",
+ "add rsi, 88064",
"mov rdx, rbx",
- "add rdx, 20480",
- "mov rcx, rbx",
- "add rcx, 3072",
- "call {vg_mldsa_expand_mask_poly}",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 27648",
+ "add rdi, 35840",
"mov rsi, rbx",
- "add rsi, 20480",
- "mov ecx, 1024",
- "26:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 26b",
+ "add rsi, 89088",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 27648",
+ "add rdi, 35840",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rax, QWORD PTR [rbx+896]",
- "add rax, 3",
- "mov BYTE PTR [rbx+1024], al",
- "shr rax, 8",
- "mov BYTE PTR [rbx+1025], al",
+ "add rsi, 90112",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 960",
- "mov esi, 524288",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 91136",
"mov rdx, rbx",
- "add rdx, 21504",
- "mov rcx, rbx",
- "add rcx, 3072",
- "call {vg_mldsa_expand_mask_poly}",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 28672",
+ "add rdi, 35840",
"mov rsi, rbx",
- "add rsi, 21504",
- "mov ecx, 1024",
- "27:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 27b",
+ "add rsi, 92160",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 28672",
+ "add rdi, 35840",
"mov rsi, rbx",
"add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rax, QWORD PTR [rbx+896]",
- "add rax, 4",
- "mov BYTE PTR [rbx+1024], al",
- "shr rax, 8",
- "mov BYTE PTR [rbx+1025], al",
+ "call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 960",
- "mov esi, 524288",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 93184",
"mov rdx, rbx",
- "add rdx, 22528",
- "mov rcx, rbx",
- "add rcx, 3072",
- "call {vg_mldsa_expand_mask_poly}",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 29696",
+ "add rdi, 36864",
"mov rsi, rbx",
- "add rsi, 22528",
- "mov ecx, 1024",
- "28:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 28b",
+ "add rsi, 94208",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 29696",
+ "add rdi, 36864",
"mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rax, QWORD PTR [rbx+896]",
- "add rax, 5",
- "mov BYTE PTR [rbx+1024], al",
- "shr rax, 8",
- "mov BYTE PTR [rbx+1025], al",
- "mov rdi, rbx",
- "add rdi, 960",
- "mov esi, 524288",
+ "add rsi, 95232",
"mov rdx, rbx",
- "add rdx, 23552",
- "mov rcx, rbx",
- "add rcx, 3072",
- "call {vg_mldsa_expand_mask_poly}",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 30720",
- "mov rsi, rbx",
- "add rsi, 23552",
- "mov ecx, 1024",
- "29:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 29b",
- "mov rdi, rbx",
- "add rdi, 30720",
- "mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rax, QWORD PTR [rbx+896]",
- "add rax, 6",
- "mov BYTE PTR [rbx+1024], al",
- "shr rax, 8",
- "mov BYTE PTR [rbx+1025], al",
- "mov rdi, rbx",
- "add rdi, 960",
- "mov esi, 524288",
- "mov rdx, rbx",
- "add rdx, 24576",
- "mov rcx, rbx",
- "add rcx, 3072",
- "call {vg_mldsa_expand_mask_poly}",
- "mov rdi, rbx",
- "add rdi, 31744",
- "mov rsi, rbx",
- "add rsi, 24576",
- "mov ecx, 1024",
- "210:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 210b",
- "mov rdi, rbx",
- "add rdi, 31744",
- "mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 32768",
- "mov rsi, rbx",
- "add rsi, 64512",
- "mov rdx, rbx",
- "add rdx, 25600",
- "call {vg_mldsa_multiply_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 32768",
- "mov rsi, rbx",
- "add rsi, 65536",
- "mov rdx, rbx",
- "add rdx, 26624",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 32768",
- "mov rsi, rbx",
- "add rsi, 66560",
- "mov rdx, rbx",
- "add rdx, 27648",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 32768",
+ "add rdi, 36864",
"mov rsi, rbx",
- "add rsi, 67584",
+ "add rsi, 96256",
"mov rdx, rbx",
"add rdx, 28672",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 32768",
+ "add rdi, 36864",
"mov rsi, rbx",
- "add rsi, 68608",
+ "add rsi, 97280",
"mov rdx, rbx",
"add rdx, 29696",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 32768",
+ "add rdi, 36864",
"mov rsi, rbx",
- "add rsi, 69632",
+ "add rsi, 98304",
"mov rdx, rbx",
"add rdx, 30720",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 32768",
+ "add rdi, 36864",
"mov rsi, rbx",
- "add rsi, 70656",
+ "add rsi, 99328",
"mov rdx, rbx",
"add rdx, 31744",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 32768",
+ "add rdi, 36864",
"mov rsi, rbx",
"add rsi, 3072",
"call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 33792",
+ "add rdi, 37888",
"mov rsi, rbx",
- "add rsi, 71680",
+ "add rsi, 100352",
"mov rdx, rbx",
"add rdx, 25600",
"call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 33792",
+ "add rdi, 37888",
"mov rsi, rbx",
- "add rsi, 72704",
+ "add rsi, 101376",
"mov rdx, rbx",
"add rdx, 26624",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 33792",
+ "add rdi, 37888",
"mov rsi, rbx",
- "add rsi, 73728",
+ "add rsi, 102400",
"mov rdx, rbx",
"add rdx, 27648",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 33792",
+ "add rdi, 37888",
"mov rsi, rbx",
- "add rsi, 74752",
+ "add rsi, 103424",
"mov rdx, rbx",
"add rdx, 28672",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 33792",
+ "add rdi, 37888",
"mov rsi, rbx",
- "add rsi, 75776",
+ "add rsi, 104448",
"mov rdx, rbx",
"add rdx, 29696",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 33792",
+ "add rdi, 37888",
"mov rsi, rbx",
- "add rsi, 76800",
+ "add rsi, 105472",
"mov rdx, rbx",
"add rdx, 30720",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 33792",
+ "add rdi, 37888",
"mov rsi, rbx",
- "add rsi, 77824",
+ "add rsi, 106496",
"mov rdx, rbx",
"add rdx, 31744",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 33792",
+ "add rdi, 37888",
"mov rsi, rbx",
"add rsi, 3072",
"call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 34816",
+ "add rdi, 38912",
"mov rsi, rbx",
- "add rsi, 78848",
+ "add rsi, 107520",
"mov rdx, rbx",
"add rdx, 25600",
"call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 34816",
+ "add rdi, 38912",
"mov rsi, rbx",
- "add rsi, 79872",
+ "add rsi, 108544",
"mov rdx, rbx",
"add rdx, 26624",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 34816",
+ "add rdi, 38912",
"mov rsi, rbx",
- "add rsi, 80896",
+ "add rsi, 109568",
"mov rdx, rbx",
"add rdx, 27648",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 34816",
+ "add rdi, 38912",
"mov rsi, rbx",
- "add rsi, 81920",
+ "add rsi, 110592",
"mov rdx, rbx",
"add rdx, 28672",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 34816",
+ "add rdi, 38912",
"mov rsi, rbx",
- "add rsi, 82944",
+ "add rsi, 111616",
"mov rdx, rbx",
"add rdx, 29696",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 34816",
+ "add rdi, 38912",
"mov rsi, rbx",
- "add rsi, 83968",
+ "add rsi, 112640",
"mov rdx, rbx",
"add rdx, 30720",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 34816",
+ "add rdi, 38912",
"mov rsi, rbx",
- "add rsi, 84992",
+ "add rsi, 113664",
"mov rdx, rbx",
"add rdx, 31744",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 34816",
+ "add rdi, 38912",
"mov rsi, rbx",
"add rsi, 3072",
"call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 35840",
+ "add rdi, 39936",
"mov rsi, rbx",
- "add rsi, 86016",
+ "add rsi, 114688",
"mov rdx, rbx",
"add rdx, 25600",
"call {vg_mldsa_multiply_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 35840",
+ "add rdi, 39936",
"mov rsi, rbx",
- "add rsi, 87040",
+ "add rsi, 115712",
"mov rdx, rbx",
"add rdx, 26624",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 35840",
+ "add rdi, 39936",
"mov rsi, rbx",
- "add rsi, 88064",
+ "add rsi, 116736",
"mov rdx, rbx",
"add rdx, 27648",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 35840",
+ "add rdi, 39936",
"mov rsi, rbx",
- "add rsi, 89088",
+ "add rsi, 117760",
"mov rdx, rbx",
"add rdx, 28672",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 35840",
+ "add rdi, 39936",
"mov rsi, rbx",
- "add rsi, 90112",
+ "add rsi, 118784",
"mov rdx, rbx",
"add rdx, 29696",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 35840",
+ "add rdi, 39936",
"mov rsi, rbx",
- "add rsi, 91136",
+ "add rsi, 119808",
"mov rdx, rbx",
"add rdx, 30720",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 35840",
+ "add rdi, 39936",
"mov rsi, rbx",
- "add rsi, 92160",
+ "add rsi, 120832",
"mov rdx, rbx",
"add rdx, 31744",
"call {vg_mldsa_multiply_add_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 35840",
+ "add rdi, 39936",
"mov rsi, rbx",
"add rsi, 3072",
"call {vg_mldsa_inv_ntt_avx2}",
"mov rdi, rbx",
- "add rdi, 36864",
- "mov rsi, rbx",
- "add rsi, 93184",
+ "add rdi, 32768",
+ "mov esi, 261888",
"mov rdx, rbx",
- "add rdx, 25600",
- "call {vg_mldsa_multiply_ntt_avx2}",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
"mov rdi, rbx",
- "add rdi, 36864",
- "mov rsi, rbx",
- "add rsi, 94208",
+ "add rdi, 6144",
+ "mov esi, 15",
"mov rdx, rbx",
- "add rdx, 26624",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 2048",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
- "add rdi, 36864",
- "mov rsi, rbx",
- "add rsi, 95232",
+ "add rdi, 33792",
+ "mov esi, 261888",
"mov rdx, rbx",
- "add rdx, 27648",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
"mov rdi, rbx",
- "add rdi, 36864",
- "mov rsi, rbx",
- "add rsi, 96256",
+ "add rdi, 6144",
+ "mov esi, 15",
"mov rdx, rbx",
- "add rdx, 28672",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 2176",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
- "add rdi, 36864",
- "mov rsi, rbx",
- "add rsi, 97280",
+ "add rdi, 34816",
+ "mov esi, 261888",
"mov rdx, rbx",
- "add rdx, 29696",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
"mov rdi, rbx",
- "add rdi, 36864",
- "mov rsi, rbx",
- "add rsi, 98304",
+ "add rdi, 6144",
+ "mov esi, 15",
"mov rdx, rbx",
- "add rdx, 30720",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 2304",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
- "add rdi, 36864",
- "mov rsi, rbx",
- "add rsi, 99328",
+ "add rdi, 35840",
+ "mov esi, 261888",
"mov rdx, rbx",
- "add rdx, 31744",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2432",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
"mov rdi, rbx",
"add rdi, 36864",
- "mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_inv_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov rsi, rbx",
- "add rsi, 100352",
- "mov rdx, rbx",
- "add rdx, 25600",
- "call {vg_mldsa_multiply_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov rsi, rbx",
- "add rsi, 101376",
- "mov rdx, rbx",
- "add rdx, 26624",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov rsi, rbx",
- "add rsi, 102400",
- "mov rdx, rbx",
- "add rdx, 27648",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov rsi, rbx",
- "add rsi, 103424",
- "mov rdx, rbx",
- "add rdx, 28672",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov rsi, rbx",
- "add rsi, 104448",
- "mov rdx, rbx",
- "add rdx, 29696",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov rsi, rbx",
- "add rsi, 105472",
- "mov rdx, rbx",
- "add rdx, 30720",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov rsi, rbx",
- "add rsi, 106496",
- "mov rdx, rbx",
- "add rdx, 31744",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_inv_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov rsi, rbx",
- "add rsi, 107520",
- "mov rdx, rbx",
- "add rdx, 25600",
- "call {vg_mldsa_multiply_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov rsi, rbx",
- "add rsi, 108544",
- "mov rdx, rbx",
- "add rdx, 26624",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov rsi, rbx",
- "add rsi, 109568",
- "mov rdx, rbx",
- "add rdx, 27648",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov rsi, rbx",
- "add rsi, 110592",
- "mov rdx, rbx",
- "add rdx, 28672",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov rsi, rbx",
- "add rsi, 111616",
- "mov rdx, rbx",
- "add rdx, 29696",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov rsi, rbx",
- "add rsi, 112640",
- "mov rdx, rbx",
- "add rdx, 30720",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov rsi, rbx",
- "add rsi, 113664",
- "mov rdx, rbx",
- "add rdx, 31744",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_inv_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov rsi, rbx",
- "add rsi, 114688",
- "mov rdx, rbx",
- "add rdx, 25600",
- "call {vg_mldsa_multiply_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov rsi, rbx",
- "add rsi, 115712",
- "mov rdx, rbx",
- "add rdx, 26624",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov rsi, rbx",
- "add rsi, 116736",
- "mov rdx, rbx",
- "add rdx, 27648",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov rsi, rbx",
- "add rsi, 117760",
- "mov rdx, rbx",
- "add rdx, 28672",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov rsi, rbx",
- "add rsi, 118784",
- "mov rdx, rbx",
- "add rdx, 29696",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov rsi, rbx",
- "add rsi, 119808",
- "mov rdx, rbx",
- "add rdx, 30720",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov rsi, rbx",
- "add rsi, 120832",
- "mov rdx, rbx",
- "add rdx, 31744",
- "call {vg_mldsa_multiply_add_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 39936",
- "mov rsi, rbx",
- "add rsi, 3072",
- "call {vg_mldsa_inv_ntt_avx2}",
- "mov rdi, rbx",
- "add rdi, 32768",
- "mov esi, 261888",
- "mov rdx, rbx",
- "add rdx, 6144",
- "call {vg_mldsa_high_bits}",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov esi, 15",
- "mov rdx, rbx",
- "add rdx, 2048",
- "mov ecx, 128",
- "call {vg_mldsa_simple_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 33792",
- "mov esi, 261888",
- "mov rdx, rbx",
- "add rdx, 6144",
- "call {vg_mldsa_high_bits}",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov esi, 15",
- "mov rdx, rbx",
- "add rdx, 2176",
- "mov ecx, 128",
- "call {vg_mldsa_simple_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 34816",
- "mov esi, 261888",
- "mov rdx, rbx",
- "add rdx, 6144",
- "call {vg_mldsa_high_bits}",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov esi, 15",
- "mov rdx, rbx",
- "add rdx, 2304",
- "mov ecx, 128",
- "call {vg_mldsa_simple_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 35840",
- "mov esi, 261888",
- "mov rdx, rbx",
- "add rdx, 6144",
- "call {vg_mldsa_high_bits}",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov esi, 15",
- "mov rdx, rbx",
- "add rdx, 2432",
- "mov ecx, 128",
- "call {vg_mldsa_simple_bit_pack}",
- "mov rdi, rbx",
- "add rdi, 36864",
- "mov esi, 261888",
- "mov rdx, rbx",
- "add rdx, 6144",
- "call {vg_mldsa_high_bits}",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
"mov rdi, rbx",
"add rdi, 6144",
"mov esi, 15",
@@ -7341,1400 +6549,608 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"add rdi, 0",
"mov esi, 136",
"mov edx, 32",
- "mov rcx, rbx",
- "add rcx, 896",
- "mov r8d, 2",
- "mov r9, rbx",
- "add r9, 200",
- "call {vg_keccak_absorb}",
- "mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 34",
- "mov ecx, 31",
- "mov r8, rbx",
- "add r8, 200",
- "call {vg_keccak_pad}",
- "mov rdi, rbx",
- "add rdi, 0",
- "mov esi, 136",
- "mov edx, 0",
- "mov rcx, rbx",
- "add rcx, 1024",
- "mov r8d, 128",
- "mov r9, rbx",
- "add r9, 200",
- "call {vg_keccak_squeeze}",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 1024",
- "mov ecx, 32",
- "20:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 20b",
- "mov rdi, rbx",
- "add rdi, 1216",
- "mov rsi, rbx",
- "add rsi, 1056",
- "mov ecx, 64",
- "21:",
- "movzx eax, BYTE PTR [rsi]",
- "mov BYTE PTR [rdi], al",
- "add rdi, 1",
- "add rsi, 1",
- "sub rcx, 1",
- "jne 21b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1281], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 4096",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 4096",
- "mov ecx, 256",
- "22:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 22b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 5120",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 5120",
- "mov ecx, 256",
- "23:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 23b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 6144",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 6144",
- "mov ecx, 256",
- "24:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 24b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 7168",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 7168",
- "mov ecx, 256",
- "25:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 25b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 8192",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 8192",
- "mov ecx, 256",
- "26:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 26b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 9216",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 9216",
- "mov ecx, 256",
- "27:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 27b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 10240",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 10240",
- "mov ecx, 256",
- "28:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 28b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 11264",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 11264",
- "mov ecx, 256",
- "29:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 29b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 12288",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 12288",
- "mov ecx, 256",
- "210:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 210b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 13312",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 13312",
- "mov ecx, 256",
- "211:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 211b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 14336",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 14336",
- "mov ecx, 256",
- "212:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 212b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 15360",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 15360",
- "mov ecx, 256",
- "213:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 213b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 16384",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 16384",
- "mov ecx, 256",
- "214:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 214b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 17408",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 17408",
- "mov ecx, 256",
- "215:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 215b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 18432",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 18432",
- "mov ecx, 256",
- "216:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 216b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 19456",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 19456",
- "mov ecx, 256",
- "217:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 217b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 20480",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 20480",
- "mov ecx, 256",
- "218:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 218b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 21504",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 21504",
- "mov ecx, 256",
- "219:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 219b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 22528",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 22528",
- "mov ecx, 256",
- "220:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 220b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 23552",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 23552",
- "mov ecx, 256",
- "221:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 221b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 24576",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 24576",
- "mov ecx, 256",
- "222:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 222b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 25600",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 25600",
- "mov ecx, 256",
- "223:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 223b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 26624",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 26624",
- "mov ecx, 256",
- "224:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 224b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 27648",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 27648",
- "mov ecx, 256",
- "225:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 225b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 28672",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 28672",
- "mov ecx, 256",
- "226:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 226b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 29696",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 29696",
- "mov ecx, 256",
- "227:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 227b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 30720",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 30720",
- "mov ecx, 256",
- "228:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 228b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 31744",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 31744",
- "mov ecx, 256",
- "229:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 229b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 32768",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 32768",
- "mov ecx, 256",
- "230:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 230b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 33792",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 33792",
- "mov ecx, 256",
- "231:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 231b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 34816",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 34816",
- "mov ecx, 256",
- "232:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 232b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 35840",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 35840",
- "mov ecx, 256",
- "233:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 233b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 36864",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 36864",
- "mov ecx, 256",
- "234:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 234b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 37888",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 37888",
- "mov ecx, 256",
- "235:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 235b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1185], al",
- "mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 38912",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 38912",
- "mov ecx, 256",
- "236:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 236b",
- "mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "mov rcx, rbx",
+ "add rcx, 896",
+ "mov r8d, 2",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
"mov rdi, rbx",
- "add rdi, 1152",
- "mov rsi, rbx",
- "add rsi, 39936",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 34",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
"mov rdi, rbx",
- "add rdi, 39936",
- "mov ecx, 256",
- "237:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
- "sub rcx, 1",
- "jne 237b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 1024",
+ "mov r8d, 128",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
"mov rdi, rbx",
"add rdi, 1152",
"mov rsi, rbx",
- "add rsi, 40960",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 40960",
- "mov ecx, 256",
- "238:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "20:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
- "jne 238b",
- "mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "jne 20b",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1216",
"mov rsi, rbx",
- "add rsi, 41984",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 41984",
- "mov ecx, 256",
- "239:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "add rsi, 1056",
+ "mov ecx, 64",
+ "21:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
- "jne 239b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "jne 21b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1281], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 43008",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 43008",
- "mov ecx, 256",
- "240:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "22:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
- "jne 240b",
- "mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "jne 22b",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1442",
"mov rsi, rbx",
- "add rsi, 44032",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 44032",
- "mov ecx, 256",
- "241:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "23:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
- "jne 241b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "jne 23b",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1476",
"mov rsi, rbx",
- "add rsi, 45056",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 45056",
- "mov ecx, 256",
- "242:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "24:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
- "jne 242b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1185], al",
+ "jne 24b",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1510",
"mov rsi, rbx",
- "add rsi, 46080",
- "mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
- "and r15d, eax",
- "mov r8d, 0",
- "sub r8d, eax",
- "mov rdi, rbx",
- "add rdi, 46080",
- "mov ecx, 256",
- "243:",
- "mov eax, DWORD PTR [rdi]",
- "and eax, r8d",
- "mov DWORD PTR [rdi], eax",
- "add rdi, 4",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "25:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
"sub rcx, 1",
- "jne 243b",
+ "jne 25b",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 47104",
+ "add rsi, 4096",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 47104",
- "mov ecx, 256",
- "244:",
+ "add rdi, 4096",
+ "mov ecx, 1024",
+ "26:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 244b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
+ "jne 26b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 48128",
+ "add rsi, 8192",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 48128",
- "mov ecx, 256",
- "245:",
+ "add rdi, 8192",
+ "mov ecx, 1024",
+ "27:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 245b",
+ "jne 27b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 49152",
+ "add rsi, 12288",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 49152",
- "mov ecx, 256",
- "246:",
+ "add rdi, 12288",
+ "mov ecx, 1024",
+ "28:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 246b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
+ "jne 28b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 50176",
+ "add rsi, 16384",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 50176",
- "mov ecx, 256",
- "247:",
+ "add rdi, 16384",
+ "mov ecx, 1024",
+ "29:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 247b",
+ "jne 29b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 51200",
+ "add rsi, 20480",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 51200",
- "mov ecx, 256",
- "248:",
+ "add rdi, 20480",
+ "mov ecx, 1024",
+ "210:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 248b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
+ "jne 210b",
"mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 52224",
+ "add rsi, 24576",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 52224",
- "mov ecx, 256",
- "249:",
+ "add rdi, 24576",
+ "mov ecx, 1024",
+ "211:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 249b",
- "mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
+ "jne 211b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 6",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 53248",
+ "add rsi, 28672",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 53248",
- "mov ecx, 256",
- "250:",
+ "add rdi, 28672",
+ "mov ecx, 1024",
+ "212:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 250b",
+ "jne 212b",
"mov eax, 0",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 54272",
+ "add rsi, 32768",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 54272",
- "mov ecx, 256",
- "251:",
+ "add rdi, 32768",
+ "mov ecx, 1024",
+ "213:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 251b",
- "mov eax, 1",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "jne 213b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 55296",
+ "add rsi, 36864",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 55296",
- "mov ecx, 256",
- "252:",
+ "add rdi, 36864",
+ "mov ecx, 1024",
+ "214:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 252b",
+ "jne 214b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1441], al",
"mov eax, 2",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 56320",
+ "add rsi, 40960",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 56320",
- "mov ecx, 256",
- "253:",
+ "add rdi, 40960",
+ "mov ecx, 1024",
+ "215:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 253b",
- "mov eax, 3",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "jne 215b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 57344",
+ "add rsi, 45056",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 57344",
- "mov ecx, 256",
- "254:",
+ "add rdi, 45056",
+ "mov ecx, 1024",
+ "216:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 254b",
+ "jne 216b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1475], al",
"mov eax, 4",
- "mov BYTE PTR [rbx+1184], al",
- "mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 58368",
+ "add rsi, 49152",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 58368",
- "mov ecx, 256",
- "255:",
+ "add rdi, 49152",
+ "mov ecx, 1024",
+ "217:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 255b",
- "mov eax, 5",
- "mov BYTE PTR [rbx+1184], al",
+ "jne 217b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1474], al",
"mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1509], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1542], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 59392",
+ "add rsi, 53248",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 59392",
- "mov ecx, 256",
- "256:",
+ "add rdi, 53248",
+ "mov ecx, 1024",
+ "218:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 256b",
+ "jne 218b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1440], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1441], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1474], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1475], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1508], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1509], al",
"mov eax, 6",
- "mov BYTE PTR [rbx+1184], al",
+ "mov BYTE PTR [rbx+1542], al",
"mov eax, 7",
- "mov BYTE PTR [rbx+1185], al",
+ "mov BYTE PTR [rbx+1543], al",
"mov rdi, rbx",
- "add rdi, 1152",
+ "add rdi, 1408",
"mov rsi, rbx",
- "add rsi, 60416",
+ "add rsi, 57344",
"mov rdx, rbx",
- "add rdx, 2048",
- "call {vg_mldsa_rej_ntt_poly}",
+ "add rdx, 79872",
+ "call {vg_mldsa_rej_ntt_poly4}",
"and r15d, eax",
"mov r8d, 0",
"sub r8d, eax",
"mov rdi, rbx",
- "add rdi, 60416",
- "mov ecx, 256",
- "257:",
+ "add rdi, 57344",
+ "mov ecx, 1024",
+ "219:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 257b",
+ "jne 219b",
"mov eax, 0",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8751,13 +7167,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 61440",
"mov ecx, 256",
- "258:",
+ "220:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 258b",
+ "jne 220b",
"mov eax, 1",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8774,13 +7190,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 62464",
"mov ecx, 256",
- "259:",
+ "221:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 259b",
+ "jne 221b",
"mov eax, 2",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8797,13 +7213,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 63488",
"mov ecx, 256",
- "260:",
+ "222:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 260b",
+ "jne 222b",
"mov eax, 3",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8820,13 +7236,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 64512",
"mov ecx, 256",
- "261:",
+ "223:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 261b",
+ "jne 223b",
"mov eax, 4",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8843,13 +7259,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 65536",
"mov ecx, 256",
- "262:",
+ "224:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 262b",
+ "jne 224b",
"mov eax, 5",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8866,13 +7282,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 66560",
"mov ecx, 256",
- "263:",
+ "225:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 263b",
+ "jne 225b",
"mov eax, 6",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8889,13 +7305,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 67584",
"mov ecx, 256",
- "264:",
+ "226:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 264b",
+ "jne 226b",
"mov eax, 7",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8912,13 +7328,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 68608",
"mov ecx, 256",
- "265:",
+ "227:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 265b",
+ "jne 227b",
"mov eax, 8",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8935,13 +7351,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 69632",
"mov ecx, 256",
- "266:",
+ "228:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 266b",
+ "jne 228b",
"mov eax, 9",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8958,13 +7374,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 70656",
"mov ecx, 256",
- "267:",
+ "229:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 267b",
+ "jne 229b",
"mov eax, 10",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -8981,13 +7397,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 71680",
"mov ecx, 256",
- "268:",
+ "230:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 268b",
+ "jne 230b",
"mov eax, 11",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -9004,13 +7420,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 72704",
"mov ecx, 256",
- "269:",
+ "231:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 269b",
+ "jne 231b",
"mov eax, 12",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -9027,13 +7443,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 73728",
"mov ecx, 256",
- "270:",
+ "232:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 270b",
+ "jne 232b",
"mov eax, 13",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -9050,13 +7466,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 74752",
"mov ecx, 256",
- "271:",
+ "233:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 271b",
+ "jne 233b",
"mov eax, 14",
"mov BYTE PTR [rbx+1280], al",
"mov rdi, rbx",
@@ -9073,49 +7489,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
"mov rdi, rbx",
"add rdi, 75776",
"mov ecx, 256",
- "272:",
+ "234:",
"mov eax, DWORD PTR [rdi]",
"and eax, r8d",
"mov DWORD PTR [rdi], eax",
"add rdi, 4",
"sub rcx, 1",
- "jne 272b",
+ "jne 234b",
"mov rdi, r12",
"add rdi, 0",
"mov rsi, rbx",
"add rsi, 1024",
"mov ecx, 32",
- "273:",
+ "235:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 273b",
+ "jne 235b",
"mov rdi, r13",
"add rdi, 0",
"mov rsi, rbx",
"add rsi, 1024",
"mov ecx, 32",
- "274:",
+ "236:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 274b",
+ "jne 236b",
"mov rdi, r13",
"add rdi, 32",
"mov rsi, rbx",
"add rsi, 1120",
"mov ecx, 32",
- "275:",
+ "237:",
"movzx eax, BYTE PTR [rsi]",
"mov BYTE PTR [rdi], al",
"add rdi, 1",
"add rsi, 1",
"sub rcx, 1",
- "jne 275b",
+ "jne 237b",
"mov rdi, rbx",
"add rdi, 61440",
"mov esi, 2",
@@ -9984,7 +8400,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk
vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
vg_keccak_pad = sym super::sha3::vg_keccak_pad,
vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
- vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4,
vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt,