diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fee5d6516..da891d7ce 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -392,8 +392,9 @@ jobs:
# and SHA-256 and SHA-512 with AVX2; and BMI2 and ADX, so X25519 and
# Ed25519 with them;
# * `hsw` (Haswell): AVX2 and BMI but not AVX-512F or ADX, so ChaCha20
- # and Poly1305 with AVX2 even where this runner has AVX-512F, and
- # X25519's and Ed25519's baselines;
+ # and Poly1305 with AVX2 even where this runner has AVX-512F,
+ # X25519's and Ed25519's baselines, and ML-DSA with its polynomial
+ # arithmetic in AVX2;
# * `p4p` (Pentium 4): nothing beyond x86-64's baseline, not even CPUID
# leaf 7, so the baseline ISA's implementations everywhere;
# * `arl` (Arrow Lake): the SHA512 extension, so SHA-512 with it, which no
@@ -409,7 +410,7 @@ jobs:
fail-fast: false
matrix:
chip: [native, skx, hsw, p4p]
- tests: [sha1 sha256 sha384 sha512 aes_gcm cmac chacha20 poly1305 mlkem x25519 ed25519 cpu]
+ tests: [sha1 sha256 sha384 sha512 aes_gcm cmac chacha20 poly1305 mlkem mldsa x25519 ed25519 cpu]
include:
- chip: arl
tests: sha384 sha512 ed25519 cpu
diff --git a/README.md b/README.md
index 96542a624..02ab01664 100644
--- a/README.md
+++ b/README.md
@@ -827,7 +827,7 @@ yours to keep:
| ✅ |
-✅ SSE2 polynomial arithmetic |
+✅ AVX2; SSE2 and AVX2 polynomial arithmetic |
✅ SHA extensions |
@@ -843,7 +843,7 @@ yours to keep:
✅ |
-✅ SSE2 polynomial arithmetic |
+✅ AVX2; SSE2 and AVX2 polynomial arithmetic |
✅ SHA extensions |
@@ -859,7 +859,7 @@ yours to keep:
✅ |
-✅ SSE2 polynomial arithmetic |
+✅ AVX2; SSE2 and AVX2 polynomial arithmetic |
✅ SHA extensions |
diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml
index 854e6670e..766d606c3 100644
--- a/docs/algorithms/ml-dsa-44.toml
+++ b/docs/algorithms/ml-dsa-44.toml
@@ -3,4 +3,4 @@ family = "Signatures"
specs = ["MlDsa"]
modules = ["src/mldsa44.rs"]
asm = ["mldsa44", "mldsa"]
-optimized = { x86_64 = "SSE2 polynomial arithmetic" }
+optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" }
diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml
index e87725611..8835d083b 100644
--- a/docs/algorithms/ml-dsa-65.toml
+++ b/docs/algorithms/ml-dsa-65.toml
@@ -3,4 +3,4 @@ family = "Signatures"
specs = ["MlDsa"]
modules = ["src/mldsa65.rs"]
asm = ["mldsa65", "mldsa"]
-optimized = { x86_64 = "SSE2 polynomial arithmetic" }
+optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" }
diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml
index 9cfbc9f3a..9a82d4299 100644
--- a/docs/algorithms/ml-dsa-87.toml
+++ b/docs/algorithms/ml-dsa-87.toml
@@ -3,4 +3,4 @@ family = "Signatures"
specs = ["MlDsa"]
modules = ["src/mldsa87.rs"]
asm = ["mldsa87", "mldsa"]
-optimized = { x86_64 = "SSE2 polynomial arithmetic" }
+optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" }
diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean
index a3d95256b..12caec852 100644
--- a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean
+++ b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean
@@ -2,6 +2,9 @@ import VerifiedGarbage.TCB.X86_64.Target
import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub
import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul
import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YNtt
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMul
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YAddSub
/-!
# ML-DSA (FIPS 204) on x86-64: the arithmetic of polynomials
@@ -80,6 +83,80 @@ def artifacts : List Artifact := [
code := Impl.MlDsa.X86_64.Arith.sub
contract := Spec.MlDsa.subContract X86_64.abi
verified := Proof.MlDsa.X86_64.Arith.sub_verified
- spSafe := Code.all_of_allInstrs (by lit_decide) }]
+ spSafe := Code.all_of_allInstrs (by lit_decide) },
+ { Spec.MlDsa.nttApi with
+ name := Spec.MlDsa.nttApi.name ++ "_avx2"
+ target := X86_64.target
+ doc := Spec.MlDsa.nttApi.doc
+ (notes := ["The function computes on eight coefficients at a time in AVX2 registers, with a table of \
+ the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications \
+ (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back \
+ before returning."])
+ code := Impl.MlDsa.X86_64.Arith.nttAvx2
+ contract := Spec.MlDsa.nttContract X86_64.abi
+ verified := Proof.MlDsa.X86_64.Arith.nttY_verified
+ spSafe := Code.all_of_allInstrs (by lit_decide)
+ features := ["avx", "avx2"]
+ ofSig := ⟨_, _, _, by unfold Spec.MlDsa.nttContract Spec.MlDsa.inPlaceContract; rfl⟩ },
+ { Spec.MlDsa.nttInvApi with
+ name := Spec.MlDsa.nttInvApi.name ++ "_avx2"
+ target := X86_64.target
+ doc := Spec.MlDsa.nttInvApi.doc
+ (notes := ["The function computes on eight coefficients at a time in AVX2 registers, with a table of \
+ the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications \
+ (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back \
+ before returning."])
+ code := Impl.MlDsa.X86_64.Arith.nttInvAvx2
+ contract := Spec.MlDsa.nttInvContract X86_64.abi
+ verified := Proof.MlDsa.X86_64.Arith.nttInvY_verified
+ spSafe := Code.all_of_allInstrs (by lit_decide)
+ features := ["avx", "avx2"]
+ ofSig := ⟨_, _, _, by unfold Spec.MlDsa.nttInvContract Spec.MlDsa.inPlaceContract; rfl⟩ },
+ { Spec.MlDsa.mulApi with
+ name := Spec.MlDsa.mulApi.name ++ "_avx2"
+ target := X86_64.target
+ doc := Spec.MlDsa.mulApi.doc
+ (notes := ["The function computes on eight coefficients at a time in AVX2 registers. It sets MXCSR to \
+ `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), \
+ through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before \
+ returning."])
+ code := Impl.MlDsa.X86_64.Arith.mulAvx2
+ contract := Spec.MlDsa.mulContract X86_64.abi
+ verified := Proof.MlDsa.X86_64.Arith.mulY_verified
+ spSafe := Code.all_of_allInstrs (by lit_decide)
+ features := ["avx", "avx2"] },
+ { Spec.MlDsa.mulAddApi with
+ name := Spec.MlDsa.mulAddApi.name ++ "_avx2"
+ target := X86_64.target
+ doc := Spec.MlDsa.mulAddApi.doc
+ (notes := ["The function computes on eight coefficients at a time in AVX2 registers. It sets MXCSR to \
+ `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), \
+ through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before \
+ returning."])
+ code := Impl.MlDsa.X86_64.Arith.mulAddAvx2
+ contract := Spec.MlDsa.mulAddContract X86_64.abi
+ verified := Proof.MlDsa.X86_64.Arith.mulAddY_verified
+ spSafe := Code.all_of_allInstrs (by lit_decide)
+ features := ["avx", "avx2"] },
+ { Spec.MlDsa.addApi with
+ name := Spec.MlDsa.addApi.name ++ "_avx2"
+ target := X86_64.target
+ doc := Spec.MlDsa.addApi.doc
+ (notes := ["The function computes on eight coefficients at a time in AVX2 registers."])
+ code := Impl.MlDsa.X86_64.Arith.addAvx2
+ contract := Spec.MlDsa.addContract X86_64.abi
+ verified := Proof.MlDsa.X86_64.Arith.addY_verified
+ spSafe := Code.all_of_allInstrs (by lit_decide)
+ features := ["avx", "avx2"] },
+ { Spec.MlDsa.subApi with
+ name := Spec.MlDsa.subApi.name ++ "_avx2"
+ target := X86_64.target
+ doc := Spec.MlDsa.subApi.doc
+ (notes := ["The function computes on eight coefficients at a time in AVX2 registers."])
+ code := Impl.MlDsa.X86_64.Arith.subAvx2
+ contract := Spec.MlDsa.subContract X86_64.abi
+ verified := Proof.MlDsa.X86_64.Arith.subY_verified
+ spSafe := Code.all_of_allInstrs (by lit_decide)
+ features := ["avx", "avx2"] }]
end VG.Artifacts.MlDsaArith.X86_64
diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean
new file mode 100644
index 000000000..4c4172ca7
--- /dev/null
+++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean
@@ -0,0 +1,203 @@
+import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Backend
+import VerifiedGarbage.Impl.MlKem.X86_64.Avx
+
+/-!
+# ML-DSA on x86-64: the polynomial arithmetic with AVX2
+
+`vg_mldsa_ntt_avx2`, `vg_mldsa_inv_ntt_avx2`, `vg_mldsa_multiply_ntt_avx2`,
+`vg_mldsa_multiply_add_ntt_avx2`, `vg_mldsa_add_avx2` and
+`vg_mldsa_sub_avx2` are their SSE2 versions (`Ntt.lean`, `Mul.lean`,
+`AddSub.lean`) on eight coefficients at a time, four in each 128-bit lane of
+AVX2 registers: in each lane, the VEX.256 form of the SSE2 code (`toY`,
+`Impl/MlKem/X86_64/Avx.lean`) does what the SSE2 code does to an `xmm`
+register, with `q`, `-q⁻¹ mod 2³²` (and `2⁶⁴ mod q`) in both lanes of
+`ymm15`, `ymm14` (and `ymm11`) (`yconsts`).
+
+The layers of the NTT and its inverse, each a pass over `f` with `rdx` at
+the coefficients it loads and `r8` at the zetas of the table:
+
+* `len ≥ 8` (`ylay`): each block with its zeta in both lanes of `ymm13`
+ (`yzeta1`), and `len / 8` times the butterflies of the eight
+ coefficients `w[j]` and the eight `w[j + len]`;
+* `len = 4` (`ylay4`): two blocks at a time, their lower halves gathered
+ into `ymm0` and their upper halves into `ymm1` by `vperm2i128`, with the
+ zeta of each in a lane of `ymm13` (`yzetaS`: `vpshufd` and `vpblendd`);
+* `len = 2` and `len = 1` (`ylay2`, `ylay1`): in each lane `l`, what
+ `vlay2` and `vlay1` do to `xmm0` and `xmm1` (or `xmm2`), with lane `l` of
+ the two loads of 32 bytes holding the coefficients at `16l` and
+ `32 + 16l`, and the zetas of the blocks there in lane `l` of `ymm13`
+ (`yzetaS`, or `vpermq` of eight zetas).
+
+`NTT⁻¹` then scales every coefficient (`yscale`). The multiplications run
+inside `withMxcsr` as in the SSE2 code: for `vg_mldsa_multiply*_ntt_avx2`,
+through the last 8 bytes of `h`, whose last eight coefficients are loaded
+to `ymm6` first; the loop stores the first 248, and the last eight are
+stored after MXCSR is loaded back. Every function clears the upper halves of
+the vector registers before returning (`vzeroupper`). Every address and
+branch depends only on the pointers.
+-/
+
+namespace VG.Impl.MlDsa.X86_64.Arith
+
+open VG.X86_64
+open VG.Impl.MlKem.X86_64 (xb xmov withMxcsr rcxLoop toY yconst)
+
+/-- `q` in the doublewords of `ymm15` and `-q⁻¹ mod 2³²` in those of `ymm14`. -/
+def yconsts : List Instr := yconst .xmm15 8380417 ++ yconst .xmm14 4236238847
+
+/-- `vzeroupper`. -/
+def yepi : List Instr := [.vop .vzeroupper]
+
+/-! ## The NTT and its inverse -/
+
+/-- The zeta at `[r8]` in every doubleword of both lanes of `ymm13`, and in
+those of `ymm12`: `vzeta 0` in each lane. -/
+def yzeta1 : List Instr :=
+ .vbroadcasti128 .xmm13 (at_ .r8 0) :: toY [.xop (.pshufd .xmm13 .xmm13 0), .xop (.pshufd .xmm12 .xmm13 0xF5)]
+
+/-- The four zetas at `[r8]` in both lanes, arranged by `vpshufd` with `o₀`
+into lane 0 of `ymm13` and with `o₁` into lane 1 (through `ymm2`), and its
+odd doublewords in the even ones of `ymm12`. -/
+def yzetaS (o₀ o₁ : BitVec 8) : List Instr :=
+ .vbroadcasti128 .xmm13 (at_ .r8 0) ::
+ toY [.xop (.pshufd .xmm2 .xmm13 o₁), .xop (.pshufd .xmm13 .xmm13 o₀)] ++
+ [.vop (.vpblendd .l256 .xmm13 .xmm13 .xmm2 0xF0)] ++ toY [.xop (.pshufd .xmm12 .xmm13 0xF5)]
+
+/-- A layer with `len ≥ 8` and butterflies `bf`: its `128 / len` blocks, the
+first with the zeta `k`, the zeta pointer moving by `dz` bytes. -/
+def ylay (bf : List Instr) (len k : Nat) (dz : BitVec 32) : Prog isa :=
+ .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k) ++
+ [.mov32 .rax (.imm (BitVec.ofNat 32 (128 / len)))])) <|
+ .loop (.seq (.block (yzeta1 ++ [.alu .add .r8 (.imm dz)]))
+ (.seq (rcxLoop (len / 8) ([.vmovdquLoad .l256 .xmm0 (at_ .rdx 0),
+ .vmovdquLoad .l256 .xmm1 (at_ .rdx (4 * len))] ++ toY bf ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx (4 * len)) .xmm3,
+ .alu .add .rdx (.imm 32)]))
+ (.block [.alu .add .rdx (.imm (BitVec.ofNat 32 (4 * len))), .alu .sub .rax (.imm 1)]))) .ne
+
+/-- The layer with `len = 4`, two blocks at a time: the first with the zeta
+`k`, the zetas of a pair at `[r8]` arranged by `yzetaS o₀ o₁`, the zeta
+pointer moving by `dz` bytes. -/
+def ylay4 (bf : List Instr) (k : Nat) (o₀ o₁ : BitVec 8) (dz : BitVec 32) : Prog isa :=
+ .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k))) <|
+ rcxLoop 16 ([.vmovdquLoad .l256 .xmm4 (at_ .rdx 0), .vmovdquLoad .l256 .xmm5 (at_ .rdx 32)] ++
+ yzetaS o₀ o₁ ++
+ [.alu .add .r8 (.imm dz), .vop (.vperm2i128 .xmm0 .xmm4 .xmm5 0x20),
+ .vop (.vperm2i128 .xmm1 .xmm4 .xmm5 0x31)] ++ toY bf ++
+ [.vop (.vperm2i128 .xmm4 .xmm0 .xmm3 0x20), .vop (.vperm2i128 .xmm5 .xmm0 .xmm3 0x31),
+ .vmovdquStore .l256 (at_ .rdx 0) .xmm4, .vmovdquStore .l256 (at_ .rdx 32) .xmm5,
+ .alu .add .rdx (.imm 64)])
+
+/-- `vlay2`'s gathering of the halves of two blocks (`Ntt.lean`). -/
+def gath2 : List Instr := [xmov .xmm2 .xmm0, xb .punpcklqdq .xmm0 .xmm1, xb .punpckhqdq .xmm2 .xmm1, xmov .xmm1 .xmm2]
+
+/-- `vlay2`'s interleaving back. -/
+def scat2 : List Instr := [xmov .xmm1 .xmm0, xb .punpcklqdq .xmm0 .xmm3, xb .punpckhqdq .xmm1 .xmm3]
+
+/-- The layer with `len = 2`, four blocks at a time: in each lane, `vlay2`'s
+two (`gath2`, `bf`, `scat2`), with their zetas from `[r8]` arranged by
+`yzetaS o₀ o₁`, the zeta pointer moving by `dz` bytes. -/
+def ylay2 (bf : List Instr) (k : Nat) (o₀ o₁ : BitVec 8) (dz : BitVec 32) : Prog isa :=
+ .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k))) <|
+ rcxLoop 16 ([.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm1 (at_ .rdx 32)] ++
+ yzetaS o₀ o₁ ++ [.alu .add .r8 (.imm dz)] ++ toY (gath2 ++ bf ++ scat2) ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1, .alu .add .rdx (.imm 64)])
+
+/-- `vlay1`'s gathering of the pairs of four blocks (`Ntt.lean`). -/
+def gath1 : List Instr :=
+ [.xop (.pshufd .xmm0 .xmm0 0xD8), .xop (.pshufd .xmm2 .xmm2 0xD8), xmov .xmm1 .xmm0,
+ xb .punpcklqdq .xmm0 .xmm2, xb .punpckhqdq .xmm1 .xmm2]
+
+/-- `vlay1`'s interleaving back. -/
+def scat1 : List Instr := [xmov .xmm1 .xmm0, xb .punpckldq .xmm0 .xmm3, xb .punpckhdq .xmm1 .xmm3]
+
+/-- The eight zetas at `[r8]` in the order the lanes of `ylay1` take them
+for `NTT`: the first two and the fifth and sixth in lane 0, the others in
+lane 1 (`vpermq`). -/
+def yzeta8 : List Instr :=
+ [.vmovdquLoad .l256 .xmm13 (at_ .r8 0), .vop (.vpermq .xmm13 .xmm13 0xD8)] ++
+ toY [.xop (.pshufd .xmm12 .xmm13 0xF5)]
+
+/-- The same for `NTT⁻¹`, whose blocks take the zetas in decreasing order:
+the eighth, seventh, fourth and third in lane 0, the others in lane 1. -/
+def yzeta8R : List Instr :=
+ [.vmovdquLoad .l256 .xmm13 (at_ .r8 0), .vop (.vpermq .xmm13 .xmm13 0x27)] ++
+ toY [.xop (.pshufd .xmm13 .xmm13 0xB1), .xop (.pshufd .xmm12 .xmm13 0xF5)]
+
+/-- The layer with `len = 1`, eight blocks at a time: in each lane,
+`vlay1`'s four, with their zetas from `[r8]` (`zl`), the zeta pointer moving
+by `dz` bytes. -/
+def ylay1 (bf : List Instr) (k : Nat) (zl : List Instr) (dz : BitVec 32) : Prog isa :=
+ .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k))) <|
+ rcxLoop 16 ([.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm2 (at_ .rdx 32)] ++ zl ++
+ [.alu .add .r8 (.imm dz)] ++ toY (gath1 ++ bf ++ scat1) ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1, .alu .add .rdx (.imm 64)])
+
+/-- Every coefficient times `256⁻¹ mod q`: `vscale` in each lane. -/
+def yscale : Prog isa :=
+ .seq (.block ([.mov .rdx (.reg .rdi)] ++ yconst .xmm13 16382 ++ [.vop (.vmovdqa .l256 .xmm12 .xmm13)]))
+ (rcxLoop 32 ([.vmovdquLoad .l256 .xmm3 (at_ .rdx 0)] ++ toY (vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++
+ vcsub .xmm3 .xmm2) ++ [.vmovdquStore .l256 (at_ .rdx 0) .xmm3, .alu .add .rdx (.imm 32)]))
+
+/-- The table and the constants. -/
+def ypro : List Instr := dwordTab zmTab 256 .rsi ++ yconsts
+
+def nttAvx2 : Prog isa := withMxcsr .rsi 768 <|
+ .seq (.block ypro) (.seq (ylay vbfly 128 1 4) (.seq (ylay vbfly 64 2 4) (.seq (ylay vbfly 32 4 4)
+ (.seq (ylay vbfly 16 8 4) (.seq (ylay vbfly 8 16 4) (.seq (ylay4 vbfly 32 0x00 0x55 8)
+ (.seq (ylay2 vbfly 64 0xA0 0xF5 16) (.seq (ylay1 vbfly 128 yzeta8 32) (.block yepi)))))))))
+
+def nttInvAvx2 : Prog isa := withMxcsr .rsi 768 <|
+ .seq (.block ypro) (.seq (ylay1 vibfly 248 yzeta8R (-32)) (.seq (ylay2 vibfly 124 0x5F 0x0A (-16))
+ (.seq (ylay4 vibfly 62 0x55 0x00 (-8)) (.seq (ylay vibfly 8 31 (-4)) (.seq (ylay vibfly 16 15 (-4))
+ (.seq (ylay vibfly 32 7 (-4)) (.seq (ylay vibfly 64 3 (-4)) (.seq (ylay vibfly 128 1 (-4))
+ (.seq yscale (.block yepi))))))))))
+
+/-! ## Products -/
+
+/-- The constants and `2⁶⁴ mod q` in the doublewords of `ymm11`. -/
+def ymulPro : List Instr := yconsts ++ yconst .xmm11 2365951
+
+/-- The coefficients of `f`, `g` and `h` to `ymm3`, `ymm13` and `ymm5`. -/
+def ymulLoads : List Instr :=
+ [.vmovdquLoad .l256 .xmm3 (at_ .rsi 0), .vmovdquLoad .l256 .xmm13 (at_ .rdx 0),
+ .vmovdquLoad .l256 .xmm5 (at_ .rdi 0)]
+
+/-- Store `ymm3` to `h` and advance the three pointers. -/
+def ymulTail : List Instr :=
+ [.vmovdquStore .l256 (at_ .rdi 0) .xmm3, .alu .add .rdi (.imm 32), .alu .add .rsi (.imm 32),
+ .alu .add .rdx (.imm 32)]
+
+/-- The last eight coefficients, with those of `h` in `ymm6`, to `ymm3`. -/
+def ymulLast (core : List Instr) : List Instr :=
+ [.vmovdquLoad .l256 .xmm3 (at_ .rsi 0), .vmovdquLoad .l256 .xmm13 (at_ .rdx 0),
+ .vop (.vmovdqa .l256 .xmm5 .xmm6)] ++ toY core
+
+/-- `mulFn` on eight coefficients at a time. -/
+def ymulFn (core : List Instr) : Prog isa :=
+ .seq (.block [.mov .r8 (.reg .rdi), .vmovdquLoad .l256 .xmm6 (at_ .rdi 992)])
+ (.seq (withMxcsr .r8 1016
+ (.seq (.block ymulPro) (.seq (rcxLoop 31 (ymulLoads ++ toY core ++ ymulTail)) (.block (ymulLast core)))))
+ (.block ([.vmovdquStore .l256 (at_ .rdi 0) .xmm3] ++ yepi)))
+
+def mulAvx2 : Prog isa := ymulFn mulCore
+
+def mulAddAvx2 : Prog isa := ymulFn mulAddCore
+
+/-! ## Sums and differences -/
+
+/-- The body of `add` and `sub` on eight coefficients at a time. -/
+def yaccBody (op : XBinOp) (fix : List Instr) : List Instr :=
+ [.vmovdquLoad .l256 .xmm0 (at_ .rdi 0), .vmovdquLoad .l256 .xmm1 (at_ .rsi 0)] ++ toY (xb op .xmm0 .xmm1 :: fix) ++
+ [.vmovdquStore .l256 (at_ .rdi 0) .xmm0, .alu .add .rdi (.imm 32), .alu .add .rsi (.imm 32)]
+
+def addAvx2 : Prog isa :=
+ .seq (.block (yconst .xmm15 8380417)) (.seq (rcxLoop 32 (yaccBody .paddd (vcsub .xmm0 .xmm2))) (.block yepi))
+
+def subAvx2 : Prog isa :=
+ .seq (.block (yconst .xmm15 8380417)) (.seq (rcxLoop 32 (yaccBody .psubd (vcadd .xmm0 .xmm2))) (.block yepi))
+
+/-- The AVX2 code. -/
+def Backend.avx2 : Backend := ⟨nttAvx2, nttInvAvx2, mulAvx2, mulAddAvx2, addAvx2, subAvx2, "_avx2"⟩
+
+end VG.Impl.MlDsa.X86_64.Arith
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean
new file mode 100644
index 000000000..157073d12
--- /dev/null
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean
@@ -0,0 +1,36 @@
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Backend
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YNtt
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMul
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YAddSub
+
+/-!
+# ML-DSA on x86-64: the polynomial arithmetic with AVX2, as an `ArithImpl`
+
+Untrusted: everything here is checked by Lean. The AVX2 code
+(`Impl/MlDsa/X86_64/Arith/Avx2.lean`) meets what the callers of the
+polynomial arithmetic need of it (`FnOk`), and requires AVX and AVX2.
+-/
+
+namespace VG.Proof.MlDsa.X86_64
+
+open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith
+
+/-- The AVX2 code. -/
+def ArithImpl.avx2 : ArithImpl where
+ code := .avx2
+ ok :=
+ { ntt := FnOk.of Arith.nttY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel)
+ (by decide +kernel)
+ invNtt := FnOk.of Arith.nttInvY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel)
+ (by decide +kernel)
+ mul := FnOk.of Arith.mulY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel)
+ (by decide +kernel)
+ mulAdd := FnOk.of Arith.mulAddY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel)
+ (by decide +kernel)
+ add := FnOk.of Arith.addY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel)
+ (by decide +kernel)
+ sub := FnOk.of Arith.subY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel)
+ (by decide +kernel) }
+ features := ["avx", "avx2"]
+
+end VG.Proof.MlDsa.X86_64
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean
index 7cdf5d3b0..dcff58350 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean
@@ -286,7 +286,7 @@ theorem fn_ok {t : Poly → Poly → Poly → Poly} {hPre : Mem → Addr → Pro
s3.gpr .rdi = coeffAddr h 252 ∧ Frame [pR h] σ.mem s3.mem ∧
(∀ k < 252, (coeffAt s3.mem h k).toNat = (R[k]!).val) ∧
∀ e < 4, (dword (s3.xmm .xmm3) e).toNat = (R[252 + e]!).val)
- fun s2 k2 f2 x2 => ?_) fun s4 ⟨s3, ⟨kk, hdi, fr, dn, ln⟩, f4, k4, x4⟩ => ?_)
+ fun s2 k2 f2 x2 _ => ?_) fun s4 ⟨s3, ⟨kk, hdi, fr, dn, ln⟩, f4, k4, x4, _⟩ => ?_)
· have fσ2 : Frame [mxH h] σ.mem s2.mem := by rw [← m1]; exact f2
refine WP.mono (WP.keep [.rax, .rdi, .rsi, .rdx, .rcx] (WP.seq (WP.mono (mulPro_ok s2)
fun w ⟨cw, xw, x6, kw, mw⟩ => ?_) (Q := fun (s3 : State) => s3.gpr .rdi = coeffAddr h 252 ∧
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean
index d87f9e044..ef50cbe5b 100644
--- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean
@@ -29,34 +29,37 @@ those bytes, and nothing more than they and MXCSR change after it. -/
theorem withMxcsrH_ok {c : Prog isa} {r : Reg} (hr : r ≠ .r11 ∧ r ≠ .rax) (rs : List Reg)
(hrs : r ∉ rs ∧ Reg.r11 ∉ rs) {p : Addr} {s : State} {Q : State → Prop}
(hsi : s.gpr r = p) (hw : pR p ∈ s.wr) (hk : writesOnly rs c = true)
- (hc : ∀ s1, Keep [.rax, .r11] s s1 → Frame [mxH p] s.mem s1.mem → s1.xmm = s.xmm → WP isa c s1 Q) :
+ (hc : ∀ s1, Keep [.rax, .r11] s s1 → Frame [mxH p] s.mem s1.mem → s1.xmm = s.xmm → s1.ymmHi = s.ymmHi →
+ WP isa c s1 Q) :
WP isa (withMxcsr r 1016 c) s fun s' =>
- ∃ s2, Q s2 ∧ Frame [mxH p] s2.mem s'.mem ∧ Keep [] s2 s' ∧ s'.xmm = s2.xmm := by
+ ∃ s2, Q s2 ∧ Frame [mxH p] s2.mem s'.mem ∧ Keep [] s2 s' ∧ s'.xmm = s2.xmm ∧ s'.ymmHi = s2.ymmHi := by
have h0 := mxH_in hw 1016 (by decide)
have h0' := mxH_in (List.mem_append_right s.rd hw) 1016 (by decide)
have h4 := mxH_in hw 1020 (by decide)
simp only [withMxcsr]
refine WP.seq (WP.mono (Q := fun (s1 : State) => s1.gpr .r11 = (s.mxcsr &&& 0xFFFF).setWidth 64 ∧ Keep [.r11] s s1 ∧
- Frame [mxH p] s.mem s1.mem ∧ s1.xmm = s.xmm) (by
+ Frame [mxH p] s.mem s1.mem ∧ s1.xmm = s.xmm ∧ s1.ymmHi = s.ymmHi) (by
vrunm [hsi, h0, h0', Mem.readW_writeW_self32, hr.1]
refine ⟨by rw [BitVec.setWidth_setWidth_of_le _ (by decide), BitVec.setWidth_eq],
⟨fun r hr => ?_, rfl, rfl⟩, (Frame.refl _ _).writeW (List.mem_singleton_self _) _
- (Offset.contains p (by decide) (by decide) (by decide))⟩
+ (Offset.contains p (by decide) (by decide) (by decide)),
+ by simp only [RegUpd.ymmHi_setReg, RegUpd.ymmHi_setFlags]⟩
simp only [List.mem_singleton] at hr
- simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false]) fun s1 ⟨h11, k1, f1, x1⟩ => ?_)
+ simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false]) fun s1 ⟨h11, k1, f1, x1, y1⟩ => ?_)
have hsi1 : s1.gpr r = p := by rw [k1.gpr (by simpa using hr.1), hsi]
have h4' : InRegions s1.wr (p + BitVec.ofNat 64 (1016 + 4)) 4 := by rw [k1.2.2]; exact h4
have h4'' : InRegions (s1.rd ++ s1.wr) (p + BitVec.ofNat 64 (1016 + 4)) 4 :=
let ⟨r, hr, hc⟩ := h4'; ⟨r, List.mem_append_right _ hr, hc⟩
refine WP.seq (WP.seq (WP.mono (Q := fun (s2 : State) => Keep [.rax] s1 s2 ∧ Frame [mxH p] s1.mem s2.mem ∧
- s2.xmm = s1.xmm)
+ s2.xmm = s1.xmm ∧ s2.ymmHi = s1.ymmHi)
(by
vrunm [hsi1, h4', h4'', Mem.readW_writeW_self32, hr.2]
refine ⟨⟨fun r hr => ?_, rfl, rfl⟩, (Frame.refl _ _).writeW (List.mem_singleton_self _) _
- (Offset.contains p (by decide) (by decide) (by decide))⟩
+ (Offset.contains p (by decide) (by decide) (by decide)), by simp only [RegUpd.ymmHi_setReg]⟩
simp only [List.mem_singleton] at hr
- simp only [RegUpd.gpr_setReg, hr, ite_false]) fun s2 ⟨k2, f2, x2⟩ => ?_))
- refine WP.seq (WP.mono (WP.keep _ (hc s2 ((k1.trans k2).mono (by simp)) (f1.trans f2) (x2.trans x1)) hk)
+ simp only [RegUpd.gpr_setReg, hr, ite_false]) fun s2 ⟨k2, f2, x2, y2⟩ => ?_))
+ refine WP.seq (WP.mono (WP.keep _ (hc s2 ((k1.trans k2).mono (by simp)) (f1.trans f2) (x2.trans x1)
+ (y2.trans y1)) hk)
fun s3 ⟨hq, k3⟩ => ?_)
have k23 := k2.trans k3
have hsi3 : s3.gpr r = p := by rw [k23.gpr (by simp [hr.2, hrs.1]), hsi1]
@@ -68,6 +71,6 @@ theorem withMxcsrH_ok {c : Prog isa} {r : Reg} (hr : r ≠ .r11 ∧ r ≠ .rax)
subst h4
vrunm [hsi3, h113, h03, h03', Mem.readW_writeW_self32, ldmxcsr_ok]
exact ⟨_, hq, (Frame.refl _ _).writeW (List.mem_singleton_self _) _
- (Offset.contains p (by decide) (by decide) (by decide)), ⟨fun _ _ => rfl, rfl, rfl⟩, rfl⟩
+ (Offset.contains p (by decide) (by decide) (by decide)), ⟨fun _ _ => rfl, rfl, rfl⟩, rfl, rfl⟩
end VG.Proof.MlDsa.X86_64.Arith
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YAddSub.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YAddSub.lean
new file mode 100644
index 000000000..701332da3
--- /dev/null
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YAddSub.lean
@@ -0,0 +1,220 @@
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YBase
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub
+
+/-!
+# ML-DSA on x86-64: `vg_mldsa_add_avx2` and `vg_mldsa_sub_avx2`
+
+Untrusted: everything here is checked by Lean. Each iteration of the loop
+loads eight coefficients of `f` and of `g` and, in each lane, does what an
+iteration of `vg_mldsa_add` (`vg_mldsa_sub`) does (`AddSub.lean`), whose
+proof holds of each lane (`ylanes`), and stores the eight results to `f`
+(`YAddSub.step`); the loop leaves `f` with all 256 (`YAddSub.fn_ok`).
+-/
+
+namespace VG.Proof.MlDsa.X86_64.Arith
+
+open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith
+open VG.Proof.MlDsa.Arith
+open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes yld_ok yconst_ok WP.keep writesOnly gprPreserved_of ifp ifn
+ ptr_step GOnly wp_rcxLoopY add_ofNat_zero lane_setReg lane_setFlags sx32 State.setMem_ymm)
+open VG.Impl.MlKem.X86_64 (xb xmov toY yconst)
+open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs)
+
+theorem addFixX_ok (s : State) (hq : s.xmm .xmm15 = qV) :
+ WP isa (.block (xb .paddd .xmm0 .xmm1 :: vcsub .xmm0 .xmm2)) s fun s' =>
+ s'.xmm .xmm0 = addV (s.xmm .xmm0) (s.xmm .xmm1) ∧ XOnly [.xmm0, .xmm2] s s' := by
+ simp only [vcsub, vcadd, xmov, xb]
+ vrun [eval_movdqa]
+ rw [hq]
+ exact ⟨rfl, by xonly⟩
+
+theorem subFixX_ok (s : State) (hq : s.xmm .xmm15 = qV) :
+ WP isa (.block (xb .psubd .xmm0 .xmm1 :: vcadd .xmm0 .xmm2)) s fun s' =>
+ s'.xmm .xmm0 = subV (s.xmm .xmm0) (s.xmm .xmm1) ∧ XOnly [.xmm0, .xmm2] s s' := by
+ simp only [vcadd, xmov, xb]
+ vrun [eval_movdqa]
+ rw [hq]
+ exact ⟨rfl, by xonly⟩
+
+theorem lane_addFix : laneSseBlock (toY (xb .paddd .xmm0 .xmm1 :: vcsub .xmm0 .xmm2)) =
+ some (xb .paddd .xmm0 .xmm1 :: vcsub .xmm0 .xmm2) := by decide +kernel
+
+theorem lane_subFix : laneSseBlock (toY (xb .psubd .xmm0 .xmm1 :: vcadd .xmm0 .xmm2)) =
+ some (xb .psubd .xmm0 .xmm1 :: vcadd .xmm0 .xmm2) := by decide +kernel
+
+namespace YAddSub
+
+/-- After `i` vectors of eight, each coefficient before `8i` is `v k`. -/
+structure Inv (s₀ : State) (v : Nat → BitVec 32) (i : Nat) (s : State) : Prop where
+ rdi : s.gpr .rdi = s₀.gpr .rdi + BitVec.ofNat 64 (32 * i)
+ rsi : s.gpr .rsi = s₀.gpr .rsi + BitVec.ofNat 64 (32 * i)
+ rd : s.rd = s₀.rd
+ wr : s.wr = s₀.wr
+ q : ∀ l < 2, s.lane .xmm15 l = qV
+ frame : Frame [pR (s₀.gpr .rdi)] s₀.mem s.mem
+ coeff : ∀ k < 256, coeffAt s.mem (s₀.gpr .rdi) k = if k < 8 * i then v k else coeffAt s₀.mem (s₀.gpr .rdi) k
+
+section
+variable {t : Poly → Poly → Poly} {s₀ : State} (hp : (accK t).pre s₀)
+ {op : XBinOp} {fix : List Instr} {F : BitVec 128 → BitVec 128 → BitVec 128}
+ {L : BitVec 32 → BitVec 32 → BitVec 32}
+ (hF : ∀ (s : State), s.xmm .xmm15 = qV →
+ WP isa (.block (xb op .xmm0 .xmm1 :: fix)) s fun s' =>
+ s'.xmm .xmm0 = F (s.xmm .xmm0) (s.xmm .xmm1) ∧ XOnly [.xmm0, .xmm2] s s')
+ (hY : laneSseBlock (toY (xb op .xmm0 .xmm1 :: fix)) = some (xb op .xmm0 .xmm1 :: fix))
+ (hL : ∀ x y : BitVec 128, ∀ e < 4, dword (F x y) e = L (dword x e) (dword y e))
+include hp hF hY hL
+
+/-- An iteration, which stores `F` of the vectors of `f` and `g` in each lane. -/
+theorem step {i : Nat} (hi : i < 32) {s : State}
+ (hI : Inv s₀ (fun k => L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) i s) :
+ WP isa (.block (yaccBody op fix ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) s fun s' =>
+ Inv s₀ (fun k => L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) (i + 1) s' ∧
+ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by
+ have j0 : 8 * i + 8 ≤ 256 := by omega
+ have hw : pR (s₀.gpr .rdi) ∈ s.wr := by rw [hI.wr, hp.2.1]; simp
+ have hr : pR (s₀.gpr .rsi) ∈ s.rd ++ s.wr := by rw [hI.rd, hI.wr, hp.1]; simp
+ have e1 : s.gpr .rdi + BitVec.ofNat 64 0 = coeffAddr (s₀.gpr .rdi) (8 * i) := by
+ rw [add_ofNat_zero, hI.rdi]; congr 2; omega
+ have e2 : s.gpr .rsi + BitVec.ofNat 64 0 = coeffAddr (s₀.gpr .rsi) (8 * i) := by
+ rw [add_ofNat_zero, hI.rsi]; congr 2; omega
+ rw [yaccBody, List.append_assoc, List.append_assoc, WP.block_append_iff,
+ show ∀ a b : Instr, [a, b] = [a] ++ [b] from fun _ _ => rfl, WP.block_append_iff]
+ refine WP.mono (yld_ok (by rw [e1]; exact f_in32 (List.mem_append_right _ hw) j0)) fun s1 ⟨L1, o1⟩ => ?_
+ refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr, e2]; exact ⟨_, hr, pR_contains32 _ j0⟩))
+ fun s2 ⟨L2, o2⟩ => ?_
+ rw [WP.block_append_iff]
+ have o12 := o1.trans o2
+ refine WP.mono (ylanes hY (P := fun l t => t.xmm .xmm0 = F ((s2.proj l).xmm .xmm0) ((s2.proj l).xmm .xmm1))
+ fun l hl => hF _ (by rw [State.proj_xmm, o12.lane _ (by decide) l hl]; exact hI.q l hl))
+ fun s3 ⟨B3, o3⟩ => ?_
+ have o13 := o12.trans o3
+ have g3 : s3.gpr .rdi = coeffAddr (s₀.gpr .rdi) (8 * i) := by rw [o13.gpr, ← e1, add_ofNat_zero]
+ have g3' : s3.gpr .rsi = coeffAddr (s₀.gpr .rsi) (8 * i) := by rw [o13.gpr, ← e2, add_ofNat_zero]
+ have w0 : InRegions s3.wr (s3.gpr .rdi) 32 := by rw [o13.wr, g3]; exact f_in32 hw j0
+ vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd,
+ State.setMem_ymm, w0, sx32]
+ refine ⟨⟨?_, ?_, ?_, ?_, fun l hl => ?_, ?_, fun k hk => ?_⟩, ?_⟩
+ · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr]
+ rw [o13.gpr, hI.rdi]; exact ptr_step _ i 32
+ · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr]
+ rw [o13.gpr, hI.rsi]; exact ptr_step _ i 32
+ · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; rw [o13.rd, hI.rd]
+ · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; rw [o13.wr, hI.wr]
+ · simp only [lane_setReg, lane_setFlags, State.setMem_lane]; rw [o13.lane _ (by decide) l hl]; exact hI.q l hl
+ · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem]
+ rw [g3, o13.mem]; exact hI.frame.writeW (List.mem_singleton_self _) _ (pR_contains32 _ j0)
+ · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem]
+ rw [g3, o13.mem, coeffAt_write256 _ _ j0 _ hk]
+ split
+ · rename_i h
+ rw [ifp (show k < 8 * (i + 1) by omega), State.ymm, extract_ymm _ _ (by omega)]
+ have hc : ∀ l < 2, ∀ e < 4, dword (s3.lane .xmm0 l) e =
+ L (coeffAt s₀.mem (s₀.gpr .rdi) (8 * i + 4 * l + e)) (coeffAt s₀.mem (s₀.gpr .rsi) (8 * i + 4 * l + e)) :=
+ fun l hl e he => by
+ rw [← State.proj_xmm, B3 l hl, hL _ _ _ he, State.proj_xmm, State.proj_xmm,
+ o2.lane _ (by decide) l hl, L1 l hl, L2 l hl, o1.gpr, o1.mem, e1, e2, dword_readW _ _ he,
+ dword_readW _ _ he, lane_load, lane_load, coeffAddr_add, coeffAddr_add, ← coeffAt_eq, ← coeffAt_eq,
+ hI.coeff _ (by omega), ifn (by omega),
+ coeffAt_frame hI.frame (by simpa using hp.2.2.1.symm) (by rw [n_eq]; omega)]
+ split
+ · rename_i h4
+ have := hc 0 (by decide) (k - 8 * i) h4
+ rw [show 8 * i + 4 * 0 + (k - 8 * i) = k by omega] at this
+ exact this
+ · rename_i h4
+ have := hc 1 (by decide) (k - 8 * i - 4) (by omega)
+ rw [show 8 * i + 4 * 1 + (k - 8 * i - 4) = k by omega] at this
+ exact this
+ · rename_i h
+ rw [hI.coeff k hk]
+ by_cases h' : k < 8 * i
+ · rw [ifp h', ifp (by omega)]
+ · rw [ifn h', ifn (by omega)]
+ · exact ⟨by rw [o13.gpr], by rw [o13.gpr]⟩
+
+/-- The whole function, from its precondition. -/
+theorem fn_ok (hv : ∀ k < 256, (L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)).toNat =
+ ((t (polyAt s₀.mem (s₀.gpr .rdi)) (polyAt s₀.mem (s₀.gpr .rsi)))[k]!).val)
+ (hc : writesOnly [.rax, .rdi, .rsi, .rcx] (.seq (.block (yconst .xmm15 8380417))
+ (.seq (VG.Impl.MlKem.X86_64.rcxLoop 32 (yaccBody op fix)) (.block yepi))) = true)
+ (hm : Code.allInstrs (fun i => !loadsMxcsr i) (.seq (.block (yconst .xmm15 8380417))
+ (.seq (VG.Impl.MlKem.X86_64.rcxLoop 32 (yaccBody op fix)) (.block yepi)) : Prog isa) = true) :
+ ∃ tr s', Exec isa (.seq (.block (yconst .xmm15 8380417))
+ (.seq (VG.Impl.MlKem.X86_64.rcxLoop 32 (yaccBody op fix)) (.block yepi))) s₀ tr s' ∧
+ abiPreserved s₀ s' ∧ (accK t).post s₀ s' := by
+ have hW : WP isa (.seq (.block (yconst .xmm15 8380417))
+ (.seq (VG.Impl.MlKem.X86_64.rcxLoop 32 (yaccBody op fix)) (.block yepi))) s₀ fun s' =>
+ Frame [pR (s₀.gpr .rdi)] s₀.mem s'.mem ∧ ∀ k < 256, coeffAt s'.mem (s₀.gpr .rdi) k =
+ L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k) := by
+ refine WP.seq (WP.mono (yconst_ok .xmm15 _ s₀) fun w ⟨lq, k1, m1, _, _⟩ => ?_)
+ refine WP.seq (WP.mono (wp_rcxLoopY (N := 32) (by decide) (by decide) _ (fun u o hy _ =>
+ ⟨by rw [o.keep.gpr (by decide), k1.gpr (by decide), Nat.mul_zero, add_ofNat_zero],
+ by rw [o.keep.gpr (by decide), k1.gpr (by decide), Nat.mul_zero, add_ofNat_zero],
+ by rw [o.keep.2.1, k1.2.1], by rw [o.keep.2.2, k1.2.2],
+ fun l hl => by
+ rw [show u.lane .xmm15 l = w.lane .xmm15 l by simp only [State.lane]; rw [o.xmm, hy], lq l hl]; decide,
+ by rw [o.mem, m1]; exact Frame.refl _ _, fun k _ => by rw [o.mem, m1, ifn (by omega)]⟩)
+ fun i hi u hI => step hp hF hY hL hi hI) fun u hI => ?_)
+ refine WP.mono (Q := fun (u' : State) => u'.mem = u.mem) (by simp only [yepi]; vrund; rfl) fun u' hm' => ?_
+ rw [hm']
+ exact ⟨hI.frame, fun k hk => by rw [hI.coeff k hk, ifp (by omega)]⟩
+ obtain ⟨tr, s', he, ⟨hf, hco⟩, hk⟩ := WP.keep _ hW hc
+ refine ⟨tr, s', he, abiPreserved_of_exec hm he (gprPreserved_of hk (by decide) hf ?_),
+ polyIs_of_toNat fun k hk => ?_⟩
+ · simpa using hp.2.2.2.1
+ · rw [n_eq] at hk
+ rw [hco k hk]
+ exact hv k hk
+
+end
+
+end YAddSub
+
+end VG.Proof.MlDsa.X86_64.Arith
+
+namespace VG.Proof.MlDsa.X86_64.Arith
+
+open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith
+open VG.Proof.MlDsa.Arith
+open VG.Spec.MlDsa (n)
+
+theorem addY_correct (s : State) (hs : (accK Spec.MlDsa.add).pre s) :
+ ∃ t s', Exec isa addAvx2 s t s' ∧ abiPreserved s s' ∧ (accK Spec.MlDsa.add).post s s' :=
+ YAddSub.fn_ok hs (op := .paddd) (fix := vcsub .xmm0 .xmm2) (L := fun a b => csubL (a + b)) addFixX_ok
+ lane_addFix (fun x y e he => dword_addV x y he)
+ (fun k hk => by
+ have hk' : k < n := by rw [n_eq]; exact hk
+ rw [add_get _ _ hk', addD_toNat (by rw [← polyAt_val hs.2.2.2.2.2.1 hk']; exact val_lt _)
+ (by rw [← polyAt_val hs.2.2.2.2.2.2 hk']; exact val_lt _),
+ ← polyAt_val hs.2.2.2.2.2.1 hk', ← polyAt_val hs.2.2.2.2.2.2 hk', val_add])
+ (by decide +kernel) (by decide +kernel)
+
+theorem subY_correct (s : State) (hs : (accK Spec.MlDsa.sub).pre s) :
+ ∃ t s', Exec isa subAvx2 s t s' ∧ abiPreserved s s' ∧ (accK Spec.MlDsa.sub).post s s' :=
+ YAddSub.fn_ok hs (op := .psubd) (fix := vcadd .xmm0 .xmm2) (L := fun a b => caddL (a - b)) subFixX_ok
+ lane_subFix (fun x y e he => dword_subV x y he)
+ (fun k hk => by
+ have hk' : k < n := by rw [n_eq]; exact hk
+ rw [sub_get _ _ hk', subD_toNat (by rw [← polyAt_val hs.2.2.2.2.2.1 hk']; exact val_lt _)
+ (by rw [← polyAt_val hs.2.2.2.2.2.2 hk']; exact val_lt _),
+ ← polyAt_val hs.2.2.2.2.2.1 hk', ← polyAt_val hs.2.2.2.2.2.2 hk', val_sub])
+ (by decide +kernel) (by decide +kernel)
+
+theorem addY_ct : ConstantTime isa (accK Spec.MlDsa.add).pre (accK Spec.MlDsa.add).pub addAvx2 :=
+ VG.Taint.constantTime (A := taint) accτ acc_agree (by taint_decide)
+
+theorem subY_ct : ConstantTime isa (accK Spec.MlDsa.sub).pre (accK Spec.MlDsa.sub).pub subAvx2 :=
+ VG.Taint.constantTime (A := taint) accτ acc_agree (by taint_decide)
+
+theorem addY_verified : Verified X86_64.target addAvx2 (Spec.MlDsa.addContract X86_64.abi) :=
+ Verified.of_correct addY_correct addY_ct (by
+ mldsa_implies [Spec.MlDsa.addContract, Spec.MlDsa.accSig, accK, X86_64.abi, X86_64.argRegs]
+ [accSat] using accSat)
+
+theorem subY_verified : Verified X86_64.target subAvx2 (Spec.MlDsa.subContract X86_64.abi) :=
+ Verified.of_correct subY_correct subY_ct (by
+ mldsa_implies [Spec.MlDsa.subContract, Spec.MlDsa.accSig, accK, X86_64.abi, X86_64.argRegs]
+ [accSat] using accSat)
+
+end VG.Proof.MlDsa.X86_64.Arith
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YBase.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YBase.lean
new file mode 100644
index 000000000..b5f4c278b
--- /dev/null
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YBase.lean
@@ -0,0 +1,130 @@
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay
+import VerifiedGarbage.Proof.MlKem.X86_64.YLanes
+import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Avx2
+
+/-!
+# ML-DSA on x86-64: coefficients in the lanes of AVX2 registers
+
+Untrusted: everything here is checked by Lean. The AVX2 code does to each
+128-bit lane what the SSE2 code does to a register (`toY`), so the proofs
+of the SSE2 code hold of each lane (`ylanes`, ML-KEM's): `YConsts` is
+`VConsts` in both lanes (`yconsts_ok`); a 256-bit load of coefficient `j`
+puts coefficients `j + 4l` to `j + 4l + 3` in lane `l` (`ylanes_load`),
+and a 256-bit store of a register whose lanes hold `a` and `a (· + 4)`
+puts `a` at coefficients `j` to `j + 7` (`polyIs_write2Y`, `ylanes_ymm`).
+-/
+
+namespace VG.Proof.MlDsa.X86_64.Arith
+
+open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith
+open VG.Proof.MlDsa.Arith
+open VG.Proof.MlKem.X86_64 (Keep XOnly XKeep YOnly ylanes yld_ok yconst_ok ifp ifn)
+open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt PolyIs)
+
+/-- `VConsts` in both lanes. -/
+def YConsts (s : State) : Prop := ∀ l < 2, VConsts (s.proj l)
+
+theorem yonly_yconsts {rs : List XReg} {s s' : State} (h : YOnly rs s s') (hc : YConsts s)
+ (h14 : XReg.xmm14 ∉ rs) (h15 : XReg.xmm15 ∉ rs) : YConsts s' := fun l hl =>
+ ⟨by rw [State.proj_xmm, h.lane _ h15 l hl]; exact (hc l hl).q,
+ by rw [State.proj_xmm, h.lane _ h14 l hl]; exact (hc l hl).qinv⟩
+
+/-- The constants in both lanes. -/
+theorem yconsts_ok (s : State) :
+ WP isa (.block yconsts) s fun s' => YConsts s' ∧ Keep [.rax] s s' ∧ s'.mem = s.mem ∧
+ s'.mxcsr = s.mxcsr ∧ ∀ r, r ≠ .xmm15 → r ≠ .xmm14 → ∀ l < 2, s'.lane r l = s.lane r l := by
+ rw [yconsts, WP.block_append_iff]
+ refine WP.mono (yconst_ok .xmm15 _ s) fun s1 ⟨l1, k1, m1, x1, o1⟩ =>
+ WP.mono (yconst_ok .xmm14 _ s1) fun s2 ⟨l2, k2, m2, x2, o2⟩ =>
+ ⟨fun l hl => ⟨?_, ?_⟩, (k1.trans k2).mono (by simp), m2.trans m1, x2.trans x1,
+ fun r h15 h14 l hl => by rw [o2 r h14 l hl, o1 r h15 l hl]⟩
+ · rw [State.proj_xmm, o2 _ (by decide) l hl, l1 l hl]; decide
+ · rw [State.proj_xmm, l2 l hl]; decide
+
+/-- The eight doublewords of a 256-bit value, as those of its lanes. -/
+theorem extract_ymm (hi lo : BitVec 128) {e : Nat} (he : e < 8) :
+ (hi ++ lo).extractLsb' (32 * e) 32 = if e < 4 then dword lo e else dword hi (e - 4) := by
+ by_cases h4 : e < 4
+ · rw [ifp h4]
+ apply BitVec.eq_of_getLsbD_eq; intro i hi'
+ simp only [dword, BitVec.getLsbD_extractLsb', hi', decide_true, Bool.true_and, BitVec.getLsbD_append,
+ ifp (show 32 * e + i < 128 by omega)]
+ · rw [ifn h4]
+ apply BitVec.eq_of_getLsbD_eq; intro i hi'
+ simp only [dword, BitVec.getLsbD_extractLsb', hi', decide_true, Bool.true_and, BitVec.getLsbD_append,
+ ifn (show ¬32 * e + i < 128 by omega)]
+ exact congrArg _ (by omega)
+
+/-- The doublewords of a 256-bit value are the coefficients `a`. -/
+def YLanes (x : BitVec 256) (a : Nat → Zq) : Prop := ∀ e < 8, (x.extractLsb' (32 * e) 32).toNat = (a e).val
+
+/-- A register whose lanes hold `a` and `a (· + 4)`. -/
+theorem ylanes_ymm {s : State} {r : XReg} {a : Nat → Zq} (h0 : DLanes (s.lane r 0) a)
+ (h1 : DLanes (s.lane r 1) (fun e => a (e + 4))) : YLanes (s.ymm r) a := fun e he => by
+ have h0' : DLanes (s.xmm r) a := h0
+ have h1' : DLanes (s.ymmHi r) (fun e => a (e + 4)) := h1
+ rw [State.ymm, extract_ymm _ _ he]
+ split
+ · exact h0' e (by omega)
+ · rw [h1' (e - 4) (by omega)]; dsimp only; rw [show e - 4 + 4 = e by omega]
+
+/-- Coefficient `i` after storing `x` at coefficient `j`. -/
+theorem coeffAt_write256 (m : Mem) (p : Addr) {j : Nat} (hj : j + 8 ≤ 256) (x : BitVec 256) {i : Nat}
+ (hi : i < 256) :
+ coeffAt (m.writeW (coeffAddr p j) x) p i =
+ if j ≤ i ∧ i < j + 8 then x.extractLsb' (32 * (i - j)) 32 else coeffAt m p i := by
+ split
+ · rename_i h
+ rw [coeffAt_eq, show coeffAddr p i = coeffAddr p j + BitVec.ofNat 64 (4 * (i - j)) by
+ rw [coeffAddr_add, show j + (i - j) = i by omega]]
+ rw [show 32 * (i - j) = 8 * (4 * (i - j)) by omega]
+ exact readW_writeW_inside (k := 4 * (i - j)) (n := 4) _ _ _ (by omega) (by decide)
+ · exact Mem.readW_writeW_sep (Offset.sep p (by omega) (by omega) (by omega)) (by decide)
+
+/-- Two vectors of eight coefficients stored into a polynomial. -/
+theorem polyIs_write2Y {m : Mem} {p : Addr} {P R : Poly} (hP : PolyIs m p P) {j j' : Nat}
+ (hj : j + 8 ≤ 256) (hj' : j' + 8 ≤ 256) (hsep : j + 8 ≤ j' ∨ j' + 8 ≤ j) {x y : BitVec 256}
+ {a b : Nat → Zq} (hx : YLanes x a) (hy : YLanes y b)
+ (hR : ∀ i < 256, R[i]! = if j ≤ i ∧ i < j + 8 then a (i - j)
+ else if j' ≤ i ∧ i < j' + 8 then b (i - j') else P[i]!) :
+ PolyIs ((m.writeW (coeffAddr p j) x).writeW (coeffAddr p j') y) p R := polyIs_of_toNat fun i hi => by
+ rw [n_eq] at hi
+ rw [coeffAt_write256 _ _ hj' _ hi, coeffAt_write256 _ _ hj _ hi, hR i hi]
+ by_cases h1 : j' ≤ i ∧ i < j' + 8
+ · rw [ite_eq_left_of_eq_true _ _ (eq_true h1), ite_eq_right_of_eq_false _ _ (eq_false (by omega)),
+ ite_eq_left_of_eq_true _ _ (eq_true h1)]
+ exact hy _ (by omega)
+ · rw [ite_eq_right_of_eq_false _ _ (eq_false h1)]
+ by_cases h2 : j ≤ i ∧ i < j + 8
+ · rw [ite_eq_left_of_eq_true _ _ (eq_true h2), ite_eq_left_of_eq_true _ _ (eq_true h2)]
+ exact hx _ (by omega)
+ · rw [ite_eq_right_of_eq_false _ _ (eq_false h2), ite_eq_right_of_eq_false _ _ (eq_false h2),
+ ite_eq_right_of_eq_false _ _ (eq_false h1)]
+ exact polyIs_toNat hP (by rw [n_eq]; exact hi)
+
+theorem pR_contains32 (p : Addr) {j : Nat} (hj : j + 8 ≤ 256) : (pR p).Contains (coeffAddr p j) 32 :=
+ Offset.contains_base p (by omega) (by omega)
+
+theorem f_in32 {rs : List Region} {fP : Addr} (hw : pR fP ∈ rs) {j : Nat} (hj : j + 8 ≤ 256) :
+ InRegions rs (coeffAddr fP j) 32 :=
+ ⟨_, hw, pR_contains32 fP hj⟩
+
+theorem frame_write2Y {m m' : Mem} {p : Addr} (hf : Frame [pR p] m m') {j j' : Nat} (hj : j + 8 ≤ 256)
+ (hj' : j' + 8 ≤ 256) (x y : BitVec 256) :
+ Frame [pR p] m ((m'.writeW (coeffAddr p j) x).writeW (coeffAddr p j') y) :=
+ (hf.writeW (List.mem_singleton_self _) x (pR_contains32 p hj)).writeW (List.mem_singleton_self _) y
+ (pR_contains32 p hj')
+
+/-- Lane `l` of a 256-bit load of coefficient `j`. -/
+theorem lane_load {p : Addr} {j l : Nat} :
+ coeffAddr p j + BitVec.ofNat 64 (16 * l) = coeffAddr p (j + 4 * l) := by
+ rw [show 16 * l = 4 * (4 * l) by omega, coeffAddr_add]
+
+/-- The lanes of a 256-bit load of coefficient `j` of `F`. -/
+theorem dlanes_loadY {m : Mem} {p : Addr} {F : Poly} (h : PolyIs m p F) {j : Nat} (hj : j + 8 ≤ 256)
+ {l : Nat} (hl : l < 2) :
+ DLanes (m.readW (coeffAddr p j + BitVec.ofNat 64 (16 * l)) 128) (fun e => F[j + 4 * l + e]!) := by
+ rw [lane_load]
+ exact dlanes_load h (by omega)
+
+end VG.Proof.MlDsa.X86_64.Arith
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay.lean
new file mode 100644
index 000000000..3468aa875
--- /dev/null
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay.lean
@@ -0,0 +1,632 @@
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YBase
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay21
+import VerifiedGarbage.Proof.MlKem.X86_64.YNtt
+import VerifiedGarbage.Proof.MlKem.X86_64.YNttLay
+
+/-!
+# ML-DSA on x86-64: the layers of the NTT and its inverse with `len ≥ 4` on AVX2 registers
+
+Untrusted: everything here is checked by Lean. For any butterfly code `bf`
+that does what `op` does to the doublewords of two SSE registers
+(`VBflyOk`) and whose AVX2 form does it in each lane
+(`laneSseBlock (toY bf) = some bf`), and any block of the specification
+whose butterflies do `op` (`BlkOk`): eight butterflies of a block
+(`ystep`), the `len / 8` of them of a block (`yblock_ok`), and the
+`128 / len` blocks of a layer with `len ≥ 8` (`ylay_ok`); and the layer
+with `len = 4`, two blocks at a time, the lower halves of their
+coefficients gathered into `ymm0` and the upper ones into `ymm1` by
+`vperm2i128` (`ylay4_ok`). The zetas: `yzeta1_ok` and `yzetaS_ok`, and the
+layers' result coefficient by coefficient (`layF_get`).
+-/
+
+namespace VG.Proof.MlDsa.X86_64.Arith
+
+open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith
+open VG.Proof.MlDsa.Arith
+open VG.Proof.MlKem.X86_64 (Keep XOnly XKeep YOnly ylanes yld_ok ifp ifn sel sel_lt sel_zero sel_55
+ add_ofNat_zero GOnly addR_ok wp_rcxLoopY wp_countdown ybcast_ok yblend_ok yperm_ok perm20 perm31
+ wp_cons_iff lane_setReg lane_setFlags State.setMem_ymm State.setMem_setMem sx32)
+open VG.Impl.MlKem.X86_64 (xb xmov toY rcxLoop)
+open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt PolyIs zetas)
+
+/-! ## A layer, coefficient by coefficient -/
+
+section
+variable {op : Zq → Zq → Zq → Zq × Zq} {blk : Poly → Nat → Nat → Nat → Nat → Poly} (hblk : BlkOk blk op)
+include hblk
+
+/-- Each coefficient after the first `b` blocks of the layer with `len`. -/
+theorem layF_get (F : Poly) {len : Nat} (hl : 0 < len) (zi : Nat → Nat) {b : Nat} (hb : 2 * len * b ≤ 256)
+ {j : Nat} (hj : j < 256) :
+ (layF blk F len zi b)[j]! = if j < 2 * len * b then
+ (if j % (2 * len) < len then (op F[j]! F[j + len]! (zetas (zi (j / (2 * len))))).1
+ else (op F[j - len]! F[j]! (zetas (zi (j / (2 * len))))).2) else F[j]! := by
+ induction b generalizing j with
+ | zero => rw [ite_eq_right (by omega)]; rfl
+ | succ b ih =>
+ have hb' : 2 * len * b + 2 * len ≤ 256 := by rw [Nat.mul_succ] at hb; exact hb
+ rw [layF, foldl_range_succ, ← layF,
+ hblk.get _ len _ _ len hl (Nat.le_refl _) (by rw [n_eq]; omega) j (by rw [n_eq]; exact hj)]
+ have hd : ∀ i, 2 * len * b ≤ i → i < 2 * len * b + 2 * len → i / (2 * len) = b ∧
+ i % (2 * len) = i - 2 * len * b := fun i h1 h2 => by
+ have e1 : i / (2 * len) = b := by
+ apply Nat.div_eq_of_lt_le
+ · rw [Nat.mul_comm]; exact h1
+ · rw [Nat.succ_mul, Nat.mul_comm b]; exact h2
+ refine ⟨e1, ?_⟩
+ have := Nat.div_add_mod i (2 * len)
+ rw [e1] at this; omega
+ by_cases h1 : 2 * len * b ≤ j ∧ j < 2 * len * b + len
+ · obtain ⟨d1, d2⟩ := hd j h1.1 (by omega)
+ rw [ite_eq_left_of_eq_true _ _ (eq_true h1), ih (by omega) hj, ih (by omega) (by omega), d1,
+ ite_eq_left_of_eq_true _ _ (eq_true (show j < 2 * len * (b + 1) by rw [Nat.mul_succ]; omega)),
+ ite_eq_left_of_eq_true _ _ (eq_true (show j % (2 * len) < len by omega)),
+ ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j < 2 * len * b by omega)),
+ ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j + len < 2 * len * b by omega))]
+ · rw [ite_eq_right_of_eq_false _ _ (eq_false h1)]
+ by_cases h2 : 2 * len * b + len ≤ j ∧ j < 2 * len * b + len + len
+ · obtain ⟨d1, d2⟩ := hd j (by omega) (by omega)
+ rw [ite_eq_left_of_eq_true _ _ (eq_true h2), ih (by omega) (by omega), ih (by omega) hj, d1,
+ ite_eq_left_of_eq_true _ _ (eq_true (show j < 2 * len * (b + 1) by rw [Nat.mul_succ]; omega)),
+ ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j % (2 * len) < len by omega)),
+ ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j - len < 2 * len * b by omega)),
+ ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j < 2 * len * b by omega))]
+ · rw [ite_eq_right_of_eq_false _ _ (eq_false h2), ih (by omega) hj]
+ by_cases h3 : j < 2 * len * b
+ · rw [ite_eq_left_of_eq_true _ _ (eq_true h3),
+ ite_eq_left_of_eq_true _ _ (eq_true (show j < 2 * len * (b + 1) by rw [Nat.mul_succ]; omega))]
+ · rw [ite_eq_right_of_eq_false _ _ (eq_false h3),
+ ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j < 2 * len * (b + 1) by rw [Nat.mul_succ]; omega))]
+
+end
+
+/-! ## Coefficients in both lanes -/
+
+/-- The facts a piece of a layer keeps. -/
+structure BInvY (fP : Addr) (s₀ s : State) : Prop where
+ keep : Keep [.r8, .rcx, .rdx, .rax] s₀ s
+ frame : Frame [pR fP] s₀.mem s.mem
+ consts : YConsts s
+ mxcsr : s.mxcsr = s₀.mxcsr
+
+theorem BInvY.trans {fP : Addr} {s₁ s₂ s₃ : State} (h₁ : BInvY fP s₁ s₂) (h₂ : BInvY fP s₂ s₃) :
+ BInvY fP s₁ s₃ :=
+ ⟨(h₁.keep.trans h₂.keep).mono (by simp), h₁.frame.trans h₂.frame, h₂.consts, h₂.mxcsr.trans h₁.mxcsr⟩
+
+/-- `YConsts` after code that changed the vector registers `rs` only, then
+nothing in them. -/
+theorem ylanes_gpr {s s' : State} (h : ∀ r l, s'.lane r l = s.lane r l) {rs : List XReg} {s₀ : State}
+ (o : YOnly rs s₀ s) (hc : YConsts s₀) (h14 : XReg.xmm14 ∉ rs) (h15 : XReg.xmm15 ∉ rs) : YConsts s' :=
+ fun l hl => ⟨by rw [State.proj_xmm, h]; exact (yonly_yconsts o hc h14 h15 l hl).q,
+ by rw [State.proj_xmm, h]; exact (yonly_yconsts o hc h14 h15 l hl).qinv⟩
+
+/-- Two registers stored into a polynomial at coefficients `j` and `j'`,
+lane `l` of each holding the four coefficients of `R` from `j + 4l` and
+`j' + 4l`. -/
+theorem polyIs_write2L {m : Mem} {p : Addr} {P R : Poly} (hP : PolyIs m p P) {j j' : Nat}
+ (hj : j + 8 ≤ 256) (hj' : j' + 8 ≤ 256) (hsep : j + 8 ≤ j' ∨ j' + 8 ≤ j) {s : State} {x y : XReg}
+ (hx : ∀ l < 2, DLanes (s.lane x l) (fun e => R[j + 4 * l + e]!))
+ (hy : ∀ l < 2, DLanes (s.lane y l) (fun e => R[j' + 4 * l + e]!))
+ (hR : ∀ i < 256, (i < j ∨ j + 8 ≤ i) → (i < j' ∨ j' + 8 ≤ i) → R[i]! = P[i]!) :
+ PolyIs ((m.writeW (coeffAddr p j) (s.ymm x)).writeW (coeffAddr p j') (s.ymm y)) p R :=
+ polyIs_write2Y hP hj hj' hsep (a := fun e => R[j + e]!) (b := fun e => R[j' + e]!)
+ (ylanes_ymm (fun e he => (hx 0 (by decide) e he).trans (by simp only [Nat.mul_zero, Nat.add_zero]))
+ (fun e he => (hx 1 (by decide) e he).trans (by dsimp only; rw [show j + 4 * 1 + e = j + (e + 4) by omega])))
+ (ylanes_ymm (fun e he => (hy 0 (by decide) e he).trans (by simp only [Nat.mul_zero, Nat.add_zero]))
+ (fun e he => (hy 1 (by decide) e he).trans (by dsimp only; rw [show j' + 4 * 1 + e = j' + (e + 4) by omega])))
+ fun i hi => by
+ by_cases h1 : j ≤ i ∧ i < j + 8
+ · rw [ite_eq_left_of_eq_true _ _ (eq_true h1), show j + (i - j) = i by omega]
+ · rw [ite_eq_right_of_eq_false _ _ (eq_false h1)]
+ by_cases h2 : j' ≤ i ∧ i < j' + 8
+ · rw [ite_eq_left_of_eq_true _ _ (eq_true h2), show j' + (i - j') = i by omega]
+ · rw [ite_eq_right_of_eq_false _ _ (eq_false h2)]; exact hR i hi (by omega) (by omega)
+
+theorem DLanes.congr {x : BitVec 128} {f g : Nat → Zq} (h : DLanes x f) (e : ∀ i < 4, f i = g i) : DLanes x g :=
+ fun i hi => by rw [h i hi, e i hi]
+
+theorem ZLanes.congr {x : BitVec 128} {f g : Nat → Zq} (h : ZLanes x f) (e : ∀ i < 4, f i = g i) : ZLanes x g :=
+ fun i hi => by rw [h i hi, e i hi]
+
+theorem getP_congr (P : Poly) {a b : Nat} (h : a = b) : P[a]! = P[b]! := h ▸ rfl
+
+/-- A butterfly of the coefficients at `a` and `b` of `P` with the zeta of
+index `zi c`, at other indices that are the same. -/
+theorem op_idx (op : Zq → Zq → Zq → Zq × Zq) (P : Poly) (zi : Nat → Nat) {a b c a' b' c' : Nat} (ha : a = a')
+ (hb : b = b') (hc : c = c') : op P[a]! P[b]! (zetas (zi c)) = op P[a']! P[b']! (zetas (zi c')) := by
+ subst ha hb hc; rfl
+
+/-- The prologue of the layers with `len` = 4, 2 and 1. -/
+theorem ypre21 {fP sP : Addr} (k : Nat) (hk : k + 4 ≤ 256) {s : State} (hdi : s.gpr .rdi = fP)
+ (hsi : s.gpr .rsi = sP) :
+ WP isa (.block (([.mov .rdx (.reg .rdi)] : List Instr) ++ leaR .r8 .rsi (4 * k))) s fun w =>
+ w.gpr .rdx = fP ∧ w.gpr .r8 = coeffAddr sP k ∧ GOnly [.rdx, .r8] s w ∧ w.ymmHi = s.ymmHi := by
+ simp only [leaR]
+ vrund [sx_ofNat (show 4 * k < 2 ^ 31 by omega), hsi, hdi]
+ exact ⟨by gonlyd, rfl⟩
+
+/-! ## The zetas -/
+
+theorem zodd_F5 (x : BitVec 128) : ZOdd x (shufDwords x 0xF5) := fun j hj => by
+ rw [dword_shufDwords _ _ (by omega)]
+ rcases (by omega : j = 0 ∨ j = 1) with rfl | rfl <;> rfl
+
+/-- The zetas `pshufd` with `o` leaves of the four at index `k` of the table. -/
+theorem zlanes_tab (o : BitVec 8) {zP : Addr} {k : Nat} (hk : ∀ j < 4, k + sel o j < 256) {m : Mem}
+ (ht : Tab zmTab m zP 256) :
+ ZLanes (shufDwords (m.readW (coeffAddr zP k) 128) o) (fun i => zetas (k + sel o i)) := fun i hi => by
+ have hs := sel_lt o i
+ rw [dword_shufDwords_sel _ _ hi, dword_readW _ _ hs, coeffAddr_add]
+ exact tab_zeta ht (hk i hi)
+
+theorem zsse1_ok (t : State) :
+ WP isa (.block [.xop (.pshufd .xmm13 .xmm13 0), .xop (.pshufd .xmm12 .xmm13 0xF5)]) t fun t' =>
+ (t'.xmm .xmm13 = shufDwords (t.xmm .xmm13) 0 ∧
+ t'.xmm .xmm12 = shufDwords (shufDwords (t.xmm .xmm13) 0) 0xF5) ∧ XOnly [.xmm13, .xmm12] t t' := by
+ vrun
+ exact ⟨by first | trivial | simp, by xonly⟩
+
+theorem yzeta1_ok {zP : Addr} {k : Nat} (hk : k + 4 ≤ 256) {s : State} (h8 : s.gpr .r8 = coeffAddr zP k)
+ (hin : InRegions (s.rd ++ s.wr) (coeffAddr zP k) 16) (ht : Tab zmTab s.mem zP 256) :
+ WP isa (.block yzeta1) s fun s' => (∀ l < 2, ZLanes (s'.lane .xmm13 l) (fun _ => zetas k) ∧
+ ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧ YOnly [.xmm13, .xmm12] s s' := by
+ rw [yzeta1, wp_cons_iff]
+ refine WP.mono (ybcast_ok (by rw [h8, add_ofNat_zero]; exact hin)) fun s1 ⟨b1, o1⟩ => ?_
+ refine WP.mono (ylanes (by decide) (P := fun l t => t.xmm .xmm13 = shufDwords (s1.lane .xmm13 l) 0 ∧
+ t.xmm .xmm12 = shufDwords (shufDwords (s1.lane .xmm13 l) 0) 0xF5)
+ fun l _ => zsse1_ok (s1.proj l)) fun s2 ⟨l2, o2⟩ => ⟨fun l hl => ?_, (o1.trans o2).mono (by simp)⟩
+ have e13 : s2.lane .xmm13 l = _ := (l2 l hl).1
+ have e12 : s2.lane .xmm12 l = _ := (l2 l hl).2
+ rw [e12, e13, b1 l hl, h8, add_ofNat_zero]
+ refine ⟨fun i hi => ?_, zodd_F5 _⟩
+ rw [zlanes_tab 0 (fun j _ => by rw [sel_zero]; omega) ht i hi]
+ dsimp only; rw [sel_zero, Nat.add_zero]
+
+theorem zsseS_ok (o₀ o₁ : BitVec 8) (t : State) :
+ WP isa (.block [.xop (.pshufd .xmm2 .xmm13 o₁), .xop (.pshufd .xmm13 .xmm13 o₀)]) t fun t' =>
+ (t'.xmm .xmm13 = shufDwords (t.xmm .xmm13) o₀ ∧ t'.xmm .xmm2 = shufDwords (t.xmm .xmm13) o₁) ∧
+ XOnly [.xmm2, .xmm13] t t' := by
+ vrun
+ exact ⟨by first | trivial | simp, by xonly⟩
+
+theorem zsseF5_ok (t : State) :
+ WP isa (.block [.xop (.pshufd .xmm12 .xmm13 0xF5)]) t fun t' =>
+ t'.xmm .xmm12 = shufDwords (t.xmm .xmm13) 0xF5 ∧ XOnly [.xmm12] t t' := by
+ vrun
+ exact ⟨by first | trivial | simp, by xonly⟩
+
+/-- The odd zetas of each lane of `ymm13` in the even doublewords of `ymm12`. -/
+theorem yF5_ok (s : State) :
+ WP isa (.block (toY [.xop (.pshufd .xmm12 .xmm13 0xF5)])) s fun s' =>
+ (∀ l < 2, s'.lane .xmm12 l = shufDwords (s.lane .xmm13 l) 0xF5) ∧ YOnly [.xmm12] s s' :=
+ ylanes (by decide) (P := fun l t => t.xmm .xmm12 = shufDwords (s.lane .xmm13 l) 0xF5)
+ fun l _ => zsseF5_ok (s.proj l)
+
+theorem yzetaS_ok (o₀ o₁ : BitVec 8) {zP : Addr} {k : Nat} (hk0 : ∀ j < 4, k + sel o₀ j < 256)
+ (hk1 : ∀ j < 4, k + sel o₁ j < 256) {s : State} (h8 : s.gpr .r8 = coeffAddr zP k)
+ (hin : InRegions (s.rd ++ s.wr) (coeffAddr zP k) 16) (ht : Tab zmTab s.mem zP 256) :
+ WP isa (.block (yzetaS o₀ o₁)) s fun s' => ZLanes (s'.lane .xmm13 0) (fun i => zetas (k + sel o₀ i)) ∧
+ ZLanes (s'.lane .xmm13 1) (fun i => zetas (k + sel o₁ i)) ∧
+ (∀ l < 2, ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧ YOnly [.xmm13, .xmm2, .xmm12] s s' := by
+ rw [yzetaS, WP.block_append_iff, WP.block_append_iff, wp_cons_iff]
+ refine WP.mono (ybcast_ok (by rw [h8, add_ofNat_zero]; exact hin)) fun s1 ⟨b1, o1⟩ => ?_
+ refine WP.mono (ylanes (by rfl) (P := fun l t =>
+ t.xmm .xmm13 = shufDwords (s1.lane .xmm13 l) o₀ ∧ t.xmm .xmm2 = shufDwords (s1.lane .xmm13 l) o₁)
+ fun l _ => zsseS_ok o₀ o₁ (s1.proj l)) fun s2 ⟨l2, o2⟩ => ?_
+ refine WP.mono (yblend_ok s2) fun s3 ⟨e0, e1, o3⟩ => ?_
+ refine WP.mono (yF5_ok s3) fun s4 ⟨f4, o4⟩ => ?_
+ have x0 : s4.lane .xmm13 0 = shufDwords (s.mem.readW (coeffAddr zP k) 128) o₀ := by
+ have e : s2.lane .xmm13 0 = _ := (l2 0 (by decide)).1
+ rw [o4.lane _ (by decide) 0 (by decide), e0, e, b1 0 (by decide), h8, add_ofNat_zero]
+ have x1 : s4.lane .xmm13 1 = shufDwords (s.mem.readW (coeffAddr zP k) 128) o₁ := by
+ have e : s2.lane .xmm2 1 = _ := (l2 1 (by decide)).2
+ rw [o4.lane _ (by decide) 1 (by decide), e1, e, b1 1 (by decide), h8, add_ofNat_zero]
+ refine ⟨by rw [x0]; exact zlanes_tab o₀ hk0 ht, by rw [x1]; exact zlanes_tab o₁ hk1 ht,
+ fun l hl => ?_, (((o1.trans o2).trans o3).trans o4).mono (by simp)⟩
+ rw [f4 l hl, o4.lane _ (by decide) l hl]
+ exact zodd_F5 _
+
+/-! ## A layer with `len ≥ 8` -/
+
+section
+variable {bf : List Instr} {op : Zq → Zq → Zq → Zq × Zq} (hbf : VBflyOk bf op)
+ (hY : laneSseBlock (toY bf) = some bf)
+ {blk : Poly → Nat → Nat → Nat → Nat → Poly} (hblk : BlkOk blk op)
+include hbf hY
+
+/-- The butterflies of `bf` in each lane, from the coefficients `x` and `y`
+of the lanes of `ymm0` and `ymm1` and the zetas `ζ` of those of `ymm13`. -/
+theorem ybf_ok {s : State} (hc : YConsts s) {x y ζ : Nat → Nat → Zq}
+ (hx : ∀ l < 2, DLanes (s.lane .xmm0 l) (x l)) (hy : ∀ l < 2, DLanes (s.lane .xmm1 l) (y l))
+ (hz : ∀ l < 2, ZLanes (s.lane .xmm13 l) (ζ l)) (ho : ∀ l < 2, ZOdd (s.lane .xmm13 l) (s.lane .xmm12 l)) :
+ WP isa (.block (toY bf)) s fun s' =>
+ (∀ l < 2, DLanes (s'.lane .xmm0 l) (fun i => (op (x l i) (y l i) (ζ l i)).1) ∧
+ DLanes (s'.lane .xmm3 l) (fun i => (op (x l i) (y l i) (ζ l i)).2)) ∧
+ YOnly [.xmm1, .xmm2, .xmm4, .xmm0, .xmm3] s s' :=
+ ylanes hY (P := fun l t => DLanes (t.xmm .xmm0) (fun i => (op (x l i) (y l i) (ζ l i)).1) ∧
+ DLanes (t.xmm .xmm3) (fun i => (op (x l i) (y l i) (ζ l i)).2))
+ fun l hl => WP.mono (hbf _ (hc l hl) _ _ _ (hx l hl) (hy l hl) (hz l hl) (ho l hl))
+ fun _ ⟨a, b, c⟩ => ⟨⟨a, b⟩, c⟩
+
+include hblk
+
+/-- The body of the loop over the vectors of a block. -/
+abbrev ybody (bf : List Instr) (len : Nat) : List Instr :=
+ [.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm1 (at_ .rdx (4 * len))] ++ toY bf ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx (4 * len)) .xmm3,
+ .alu .add .rdx (.imm 32)] ++ [.alu .sub .rcx (.imm 1)]
+
+theorem ystep {fP : Addr} {len st u k : Nat} (hl : 8 ≤ len) (hs : st + 2 * len ≤ 256) (hu : 8 * u + 8 ≤ len)
+ {G : Poly} {s : State} (hc : YConsts s) (hz : ∀ l < 2, ZLanes (s.lane .xmm13 l) (fun _ => zetas k))
+ (ho : ∀ l < 2, ZOdd (s.lane .xmm13 l) (s.lane .xmm12 l))
+ (hdx : s.gpr .rdx = coeffAddr fP (st + 8 * u)) (hS : PolyIs s.mem fP (blk G len k st (8 * u)))
+ (hw : pR fP ∈ s.wr) :
+ WP isa (.block (ybody bf len)) s fun s' =>
+ PolyIs s'.mem fP (blk G len k st (8 * (u + 1))) ∧ s'.gpr .rdx = coeffAddr fP (st + 8 * (u + 1)) ∧
+ Frame [pR fP] s.mem s'.mem ∧ YConsts s' ∧ (∀ l < 2, s'.lane .xmm13 l = s.lane .xmm13 l) ∧
+ (∀ l < 2, s'.lane .xmm12 l = s.lane .xmm12 l) ∧ Keep [.rdx, .rcx] s s' ∧
+ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) ∧ s'.mxcsr = s.mxcsr := by
+ have j0 : st + 8 * u + 8 ≤ 256 := by omega
+ have j1 : st + 8 * u + len + 8 ≤ 256 := by omega
+ have a1 : coeffAddr fP (st + 8 * u) + BitVec.ofNat 64 (4 * len) = coeffAddr fP (st + 8 * u + len) :=
+ coeffAddr_add _ _ _
+ have r0 : InRegions (s.rd ++ s.wr) (coeffAddr fP (st + 8 * u) + BitVec.ofNat 64 0) 32 := by
+ rw [add_ofNat_zero]; exact f_in32 (List.mem_append_right _ hw) j0
+ have r1 : InRegions (s.rd ++ s.wr) (coeffAddr fP (st + 8 * u) + BitVec.ofNat 64 (4 * len)) 32 := by
+ rw [a1]; exact f_in32 (List.mem_append_right _ hw) j1
+ rw [ybody, List.append_assoc, List.append_assoc, WP.block_append_iff,
+ show ∀ a b : Instr, [a, b] = [a] ++ [b] from fun _ _ => rfl, WP.block_append_iff]
+ refine WP.mono (yld_ok (by rw [hdx]; exact r0)) fun s1 ⟨L1, o1⟩ => ?_
+ refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr, hdx]; exact r1)) fun s2 ⟨L2, o2⟩ => ?_
+ rw [WP.block_append_iff]
+ have o12 := o1.trans o2
+ generalize hP : blk G len k st (8 * u) = P at hS
+ refine WP.mono (ybf_ok hbf hY (yonly_yconsts o12 hc (by decide) (by decide))
+ (x := fun l e => P[st + 8 * u + 4 * l + e]!) (y := fun l e => P[st + 8 * u + len + 4 * l + e]!)
+ (ζ := fun _ _ => zetas k)
+ (fun l hl => by
+ rw [o2.lane _ (by decide) l hl, L1 l hl, hdx, add_ofNat_zero]; exact dlanes_loadY hS j0 hl)
+ (fun l hl => by
+ rw [L2 l hl, o1.gpr, o1.mem, hdx, a1]; exact dlanes_loadY hS j1 hl)
+ (fun l hl => by rw [o12.lane _ (by decide) l hl]; exact hz l hl)
+ (fun l hl => by rw [o12.lane _ (by decide) l hl, o12.lane _ (by decide) l hl]; exact ho l hl))
+ fun s3 ⟨B3, o3⟩ => ?_
+ have o13 := o12.trans o3
+ have w0 : InRegions s3.wr (s3.gpr .rdx) 32 := by
+ rw [o13.wr, o13.gpr, hdx]; exact f_in32 hw j0
+ have w1 : InRegions s3.wr (s3.gpr .rdx + BitVec.ofNat 64 (4 * len)) 32 := by
+ rw [o13.wr, o13.gpr, hdx, a1]; exact f_in32 hw j1
+ vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, State.setMem_ymm,
+ State.setMem_setMem, w0, w1, sx32]
+ have g3 : s3.gpr .rdx = coeffAddr fP (st + 8 * u) := by rw [o13.gpr, hdx]
+ rw [g3, a1, o13.mem]
+ refine ⟨?_, ?_, ?_, ?_, ?_, ?_, ⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, ?_⟩
+ · have hR : ∀ i < 256, (blk G len k st (8 * (u + 1)))[i]! = if st + 8 * u ≤ i ∧ i < st + 8 * u + 8 then
+ (op P[i]! P[i + len]! (zetas k)).1 else if st + 8 * u + len ≤ i ∧ i < st + 8 * u + len + 8 then
+ (op P[i - len]! P[i]! (zetas k)).2 else P[i]! := fun i hi => by
+ rw [← hP, show 8 * (u + 1) = 8 * u + 8 by omega, hblk.add, hblk.get _ _ _ _ _ (by omega) (by omega)
+ (by rw [n_eq]; omega) _ (by rw [n_eq]; exact hi)]
+ refine polyIs_write2L (s := s3) hS j0 j1 (by omega) (fun l hl e he => ?_) (fun l hl e he => ?_)
+ (fun i hi h1 h2 => by rw [hR i hi, ite_eq_right (by omega), ite_eq_right (by omega)])
+ · rw [(B3 l hl).1 e he]; dsimp only; rw [hR _ (by omega), ite_eq_left (by omega),
+ show st + 8 * u + len + 4 * l + e = st + 8 * u + 4 * l + e + len by omega]
+ · rw [(B3 l hl).2 e he]; dsimp only; rw [hR _ (by omega), ite_eq_right (by omega), ite_eq_left (by omega),
+ show st + 8 * u + len + 4 * l + e - len = st + 8 * u + 4 * l + e by omega]
+ · rw [show (32 : BitVec 64) = BitVec.ofNat 64 (4 * 8) from rfl, coeffAddr_add,
+ show st + 8 * u + 8 = st + 8 * (u + 1) by omega]
+ · exact frame_write2Y (Frame.refl _ _) j0 j1 _ _
+ · exact ylanes_gpr (s := s3) (fun r l => by simp only [lane_setReg, lane_setFlags, State.setMem_lane]) o13 hc
+ (by decide) (by decide)
+ · intro l hl; simp only [lane_setReg, lane_setFlags, State.setMem_lane]; exact o13.lane _ (by decide) l hl
+ · intro l hl; simp only [lane_setReg, lane_setFlags, State.setMem_lane]; exact o13.lane _ (by decide) l hl
+ · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr
+ simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr.1, hr.2, ite_false, State.setMem_gpr]
+ rw [o13.gpr]
+ · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; rw [o13.rd]
+ · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; rw [o13.wr]
+ · rw [o13.gpr]
+ · rw [o13.gpr]
+ · exact o13.mxcsr
+
+/-- The code of a block of a layer with `len ≥ 8`. -/
+abbrev yblk (bf : List Instr) (len : Nat) (dz : BitVec 32) : Prog isa :=
+ .seq (.block (yzeta1 ++ [.alu .add .r8 (.imm dz)]))
+ (.seq (rcxLoop (len / 8) ([.vmovdquLoad .l256 .xmm0 (at_ .rdx 0),
+ .vmovdquLoad .l256 .xmm1 (at_ .rdx (4 * len))] ++ toY bf ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx (4 * len)) .xmm3,
+ .alu .add .rdx (.imm 32)]))
+ (.block [.alu .add .rdx (.imm (BitVec.ofNat 32 (4 * len))), .alu .sub .rax (.imm 1)]))
+
+theorem yblock_ok {fP sP : Addr} {len st kz : Nat} (h8 : 8 ≤ len) (hl8 : len % 8 = 0) (hl : len ≤ 128)
+ (hs : st + 2 * len ≤ 256) (hkz : kz + 4 ≤ 256) (dz : BitVec 32) {G : Poly} {s : State} (hc : YConsts s)
+ (hdx : s.gpr .rdx = coeffAddr fP st) (h8r : s.gpr .r8 = coeffAddr sP kz) (hS : PolyIs s.mem fP G)
+ (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (yblk bf len dz) s fun s' => PolyIs s'.mem fP (blk G len kz st len) ∧
+ s'.gpr .rdx = coeffAddr fP (st + 2 * len) ∧ s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧
+ s'.gpr .rax = s.gpr .rax - 1 ∧ s'.zf = some (s.gpr .rax - 1 == 0) ∧ BInvY fP s s' := by
+ -- the zeta
+ refine WP.seq ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (yzeta1_ok hkz h8r (tab_in (List.mem_append_right _ hw) hkz) hT) fun s1 ⟨z1, o1⟩ => ?_
+ have g1 : s1.gpr = s.gpr := o1.gpr
+ refine WP.mono (addR_ok .r8 dz s1) fun s2 ⟨h82, o2, y2⟩ => ?_
+ have l2 := o2.lane y2
+ have c2 : YConsts s2 := ylanes_gpr (s := s1) l2 o1 hc (by decide) (by decide)
+ have dx2 : s2.gpr .rdx = coeffAddr fP st := by rw [o2.keep.gpr (by decide), g1, hdx]
+ have hw2 : pR fP ∈ s2.wr := by rw [o2.keep.2.2, o1.wr]; exact hwf
+ have m2 : s2.mem = s.mem := by rw [o2.mem, o1.mem]
+ refine WP.seq (WP.mono (wp_rcxLoopY (N := len / 8) (by omega) (by omega)
+ (fun u w => PolyIs w.mem fP (blk G len kz st (8 * u)) ∧ w.gpr .rdx = coeffAddr fP (st + 8 * u) ∧
+ YConsts w ∧ (∀ l < 2, w.lane .xmm13 l = s2.lane .xmm13 l) ∧ (∀ l < 2, w.lane .xmm12 l = s2.lane .xmm12 l) ∧
+ Keep [.rcx, .rdx] s2 w ∧ Frame [pR fP] s2.mem w.mem ∧ w.mxcsr = s2.mxcsr)
+ (fun w o hy _ => ⟨by rw [hblk.zero, o.mem, m2]; exact hS, by rw [o.keep.gpr (by decide), dx2]; rfl,
+ ylanes_gpr (s := s2) (o.lane hy) (YOnly.refl [] s2) c2 (by decide) (by decide),
+ fun l _ => o.lane hy _ l, fun l _ => o.lane hy _ l, o.keep.mono (by simp),
+ by rw [o.mem]; exact Frame.refl _ _, o.mxcsr⟩)
+ (fun u hu w ⟨hS', hdx', hc', hz', hzo', hk', hf', hx'⟩ => WP.mono (ystep hbf hY hblk h8 hs (by
+ have := Nat.div_mul_cancel (Nat.dvd_of_mod_eq_zero hl8); omega) hc'
+ (fun l hl => by rw [hz' l hl, l2]; exact (z1 l hl).1)
+ (fun l hl => by rw [hz' l hl, hzo' l hl, l2, l2]; exact (z1 l hl).2) hdx' hS'
+ (by rw [hk'.2.2]; exact hw2))
+ fun w' ⟨hS'', hdx'', hf'', hc'', hz'', hzo'', hk'', hcx, hzf, hx''⟩ =>
+ ⟨⟨hS'', hdx'', hc'', fun l hl => by rw [hz'' l hl, hz' l hl], fun l hl => by rw [hzo'' l hl, hzo' l hl],
+ (hk'.trans hk'').mono (by simp), hf'.trans hf'', by rw [hx'', hx']⟩, hcx, hzf⟩))
+ fun w ⟨hS3, hdx3, hc3, _, _, hk3, hf3, hx3⟩ => ?_)
+ rw [show 8 * (len / 8) = len from Nat.mul_div_cancel' (Nat.dvd_of_mod_eq_zero hl8)] at hS3 hdx3
+ have hax : w.gpr .rax = s.gpr .rax := by rw [hk3.gpr (by decide), o2.keep.gpr (by decide), g1]
+ have h8w : w.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz := by rw [hk3.gpr (by decide), h82, g1]
+ vrund [hdx3, sx_ofNat (show 4 * len < 2 ^ 31 by omega), hax, h8w]
+ refine ⟨hS3, by rw [coeffAddr_add, show st + len + len = st + 2 * len by omega], ?_⟩
+ have k1 : Keep [.r8, .rcx, .rdx, .rax] s w :=
+ (Keep.trans (⟨fun r _ => by rw [g1], o1.rd, o1.wr⟩ : Keep [] s s1) (o2.keep.trans hk3)).mono (by simp)
+ refine ⟨⟨fun r hr => ?_, k1.2.1, k1.2.2⟩, by rw [← m2]; exact hf3,
+ ylanes_gpr (s := w) (fun r l => by simp only [lane_setReg, lane_setFlags]) (YOnly.refl [] w) hc3
+ (by decide) (by decide),
+ by simp only [RegUpd.mxcsr_setReg, RegUpd.mxcsr_setFlags]; rw [hx3, o2.mxcsr, o1.mxcsr]⟩
+ simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr
+ simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false]
+ exact k1.gpr (by simp [hr])
+
+theorem ylay_ok {fP sP : Addr} {len k : Nat} (hlen : len ∈ [8, 16, 32, 64, 128]) (dz : BitVec 32)
+ (zi : Nat → Nat) (hz0 : zi 0 = k) (hzi : ∀ c < 128 / len, zi c + 4 ≤ 256)
+ (hstep : ∀ c < 128 / len, coeffAddr sP (zi c) + BitVec.signExtend 64 dz = coeffAddr sP (zi (c + 1)))
+ {F : Poly} {s : State} (hc : YConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP)
+ (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr)
+ (hd : (pR sP).Disjoint (pR fP)) :
+ WP isa (ylay bf len k dz) s fun s' => PolyIs s'.mem fP (layF blk F len zi (128 / len)) ∧
+ BInvY fP s s' := by
+ have hl : 8 ≤ len ∧ len % 8 = 0 ∧ len ≤ 128 ∧ 2 * len * (128 / len) = 256 ∧ 0 < 128 / len ∧
+ 128 / len ≤ 16 := by
+ simp only [List.mem_cons, List.not_mem_nil, or_false] at hlen
+ rcases hlen with rfl | rfl | rfl | rfl | rfl <;> decide
+ obtain ⟨h8, hl8, hl128, hcov, hpos, h16⟩ := hl
+ have hk : k + 4 ≤ 256 := hz0 ▸ hzi 0 hpos
+ refine WP.seq (WP.mono (Q := fun (w : State) => w.gpr .rdx = fP ∧ w.gpr .r8 = coeffAddr sP k ∧
+ w.gpr .rax = BitVec.ofNat 64 (128 / len) ∧ GOnly [.rdx, .r8, .rax] s w ∧ w.ymmHi = s.ymmHi)
+ (by
+ simp only [leaR]
+ vrund [sx_ofNat (show 4 * k < 2 ^ 31 by omega), hsi, hdi, RegUpd.ymmHi_setReg]
+ refine ⟨?_, by gonlyd, rfl⟩
+ apply BitVec.eq_of_toNat_eq
+ rw [BitVec.toNat_setWidth, BitVec.toNat_ofNat, BitVec.toNat_ofNat]
+ omega) fun w ⟨hdx, h8r, hax, o, hy⟩ => ?_)
+ have hwf' : pR fP ∈ w.wr := by rw [o.keep.2.2]; exact hwf
+ have hw' : pR sP ∈ w.wr := by rw [o.keep.2.2]; exact hw
+ have cw : YConsts w := ylanes_gpr (s := s) (o.lane hy) (YOnly.refl [] s) hc (by decide) (by decide)
+ refine WP.mono (wp_countdown (cnt := .rax) (N := 128 / len) (by omega) hpos
+ (fun c u => PolyIs u.mem fP (layF blk F len zi c) ∧ u.gpr .rdx = coeffAddr fP (2 * len * c) ∧
+ u.gpr .r8 = coeffAddr sP (zi c) ∧ BInvY fP w u ∧ Tab zmTab u.mem sP 256)
+ (fun c hc u ⟨hS', hdx', h8', hb', hT'⟩ _ => ?_) (fun u h => h)
+ ⟨by rw [o.mem]; exact hS, by rw [hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero],
+ by rw [h8r, hz0], ⟨Keep.refl _ _, Frame.refl _ _, cw, rfl⟩, by rw [o.mem]; exact hT⟩ hax)
+ fun u ⟨hS', _, _, hb', _⟩ => ⟨hS', ⟨(o.keep.trans hb'.keep).mono (by simp),
+ by rw [← o.mem]; exact hb'.frame, hb'.consts, by rw [hb'.mxcsr, o.mxcsr]⟩⟩
+ have hs : 2 * len * c + 2 * len ≤ 256 := by
+ have : 2 * len * (c + 1) ≤ 2 * len * (128 / len) := Nat.mul_le_mul_left _ (by omega)
+ rw [Nat.mul_succ] at this; omega
+ refine WP.mono (yblock_ok hbf hY hblk h8 hl8 hl128 hs (hzi c hc) dz hb'.consts hdx' h8' hS' hT'
+ (by rw [hb'.keep.2.2]; exact hwf') (by rw [hb'.keep.2.2]; exact hw'))
+ fun u' ⟨hS'', hdx'', h8'', hax'', hzf'', hb''⟩ =>
+ ⟨⟨by rw [layF, foldl_range_succ]; exact hS'', by rw [hdx'', Nat.mul_succ],
+ by rw [h8'', h8', hstep c hc], hb'.trans hb'',
+ hT'.frame hb''.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide)⟩, hax'', hzf''⟩
+
+/-! ## The layer with `len = 4` -/
+
+/-- The body of the layer with `len = 4`. -/
+abbrev ybody4 (bf : List Instr) (o₀ o₁ : BitVec 8) (dz : BitVec 32) : List Instr :=
+ [.vmovdquLoad .l256 .xmm4 (at_ .rdx 0)] ++ ([.vmovdquLoad .l256 .xmm5 (at_ .rdx 32)] ++ (yzetaS o₀ o₁ ++
+ ([.alu .add .r8 (.imm dz)] ++ ([.vop (.vperm2i128 .xmm0 .xmm4 .xmm5 0x20)] ++
+ ([.vop (.vperm2i128 .xmm1 .xmm4 .xmm5 0x31)] ++ (toY bf ++ ([.vop (.vperm2i128 .xmm4 .xmm0 .xmm3 0x20)] ++
+ ([.vop (.vperm2i128 .xmm5 .xmm0 .xmm3 0x31)] ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm4, .vmovdquStore .l256 (at_ .rdx 32) .xmm5, .alu .add .rdx (.imm 64),
+ .alu .sub .rcx (.imm 1)]))))))))
+
+theorem ystep4 {fP sP : Addr} {m kb : Nat} (hm : m < 16) (o₀ o₁ : BitVec 8) (dz : BitVec 32) (zi : Nat → Nat)
+ (hk : kb + 4 ≤ 256) (hsel : ∀ e < 4, kb + sel o₀ e = zi (2 * m) ∧ kb + sel o₁ e = zi (2 * m + 1))
+ {F : Poly} {s : State} (hc : YConsts s) (hdx : s.gpr .rdx = coeffAddr fP (16 * m))
+ (h8 : s.gpr .r8 = coeffAddr sP kb) (hS : PolyIs s.mem fP (layF blk F 4 zi (2 * m)))
+ (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (.block (ybody4 bf o₀ o₁ dz)) s fun s' =>
+ PolyIs s'.mem fP (layF blk F 4 zi (2 * (m + 1))) ∧ s'.gpr .rdx = coeffAddr fP (16 * (m + 1)) ∧
+ s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧
+ s'.zf = some (s.gpr .rcx - 1 == 0) ∧ BInvY fP s s' := by
+ have j0 : 16 * m + 8 ≤ 256 := by omega
+ have j1 : 16 * m + 8 + 8 ≤ 256 := by omega
+ have a1 : coeffAddr fP (16 * m) + BitVec.ofNat 64 32 = coeffAddr fP (16 * m + 8) := coeffAddr_add _ _ 8
+ have r0 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 0) 32 := by
+ rw [hdx, add_ofNat_zero]; exact f_in32 (List.mem_append_right s.rd hwf) j0
+ have r1 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 32) 32 := by
+ rw [hdx, a1]; exact f_in32 (List.mem_append_right s.rd hwf) j1
+ generalize hP : layF blk F 4 zi (2 * m) = P at hS
+ -- the loads
+ rw [ybody4, WP.block_append_iff]
+ refine WP.mono (yld_ok r0) fun s1 ⟨L4, o1⟩ => ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr]; exact r1)) fun s2 ⟨L5, o2⟩ => ?_
+ have o12 := o1.trans o2
+ -- the zetas
+ rw [WP.block_append_iff]
+ have hk0 : ∀ j < 4, kb + sel o₀ j < 256 := fun j _ => by have := sel_lt o₀ j; omega
+ have hk1 : ∀ j < 4, kb + sel o₁ j < 256 := fun j _ => by have := sel_lt o₁ j; omega
+ refine WP.mono (yzetaS_ok o₀ o₁ (zP := sP) (k := kb) hk0 hk1 (by rw [o12.gpr, h8])
+ (by rw [o12.rd, o12.wr]; exact tab_in (List.mem_append_right _ hw) hk) (by rw [o12.mem]; exact hT))
+ fun s3 ⟨Z0, Z1, ZO, o3⟩ => ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (addR_ok .r8 dz s3) fun s4 ⟨h84, g4, y4⟩ => ?_
+ have l4 := g4.lane y4
+ -- the lower and upper halves
+ rw [WP.block_append_iff]
+ refine WP.mono (yperm_ok s4) fun s5 ⟨P5, o5⟩ => ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (yperm_ok s5) fun s6 ⟨P6, o6⟩ => ?_
+ have c6 : YConsts s6 := yonly_yconsts (o5.trans o6)
+ (ylanes_gpr (s := s3) l4 (o12.trans o3) hc (by decide) (by decide)) (by decide) (by decide)
+ have m3 : s3.mem = s.mem := (o12.trans o3).mem
+ have q4 : ∀ l < 2, s4.lane .xmm4 l = s.mem.readW (coeffAddr fP (16 * m) + BitVec.ofNat 64 (16 * l)) 128 :=
+ fun l hl => by rw [l4, o3.lane _ (by decide) l hl, o2.lane _ (by decide) l hl, L4 l hl, hdx, add_ofNat_zero]
+ have q5 : ∀ l < 2, s4.lane .xmm5 l =
+ s.mem.readW (coeffAddr fP (16 * m + 8) + BitVec.ofNat 64 (16 * l)) 128 :=
+ fun l hl => by rw [l4, o3.lane _ (by decide) l hl, L5 l hl, o1.gpr, o1.mem, hdx, a1]
+ rw [WP.block_append_iff]
+ refine WP.mono (ybf_ok hbf hY c6 (x := fun l e => P[16 * m + 8 * l + e]!)
+ (y := fun l e => P[16 * m + 8 * l + 4 + e]!) (ζ := fun l _ => zetas (zi (2 * m + l)))
+ (fun l hl => by
+ rw [o6.lane _ (by decide) l hl, P5 l hl, perm20 _ _ hl]
+ rcases lane01 hl with rfl | rfl
+ · rw [ifp rfl, q4 0 (by decide)]
+ exact (dlanes_loadY hS j0 (by decide)).congr fun e _ => getP_congr P (by omega)
+ · rw [ifn (by decide), q5 0 (by decide)]
+ exact (dlanes_loadY hS j1 (by decide)).congr fun e _ => getP_congr P (by omega))
+ (fun l hl => by
+ rw [P6 l hl, perm31 _ _ hl, o5.lane .xmm4 (by decide) 1 (by decide), o5.lane .xmm5 (by decide) 1 (by decide)]
+ rcases lane01 hl with rfl | rfl
+ · rw [ifp rfl, q4 1 (by decide)]
+ exact (dlanes_loadY hS j0 (by decide)).congr fun e _ => getP_congr P (by omega)
+ · rw [ifn (by decide), q5 1 (by decide)]
+ exact (dlanes_loadY hS j1 (by decide)).congr fun e _ => getP_congr P (by omega))
+ (fun l hl => by
+ rw [(o5.trans o6).lane _ (by decide) l hl, l4]
+ rcases lane01 hl with rfl | rfl
+ · exact Z0.congr fun i hi => congrArg zetas (hsel i hi).1
+ · exact Z1.congr fun i hi => congrArg zetas (hsel i hi).2)
+ (fun l hl => by
+ rw [(o5.trans o6).lane _ (by decide) l hl, (o5.trans o6).lane _ (by decide) l hl, l4, l4]
+ exact ZO l hl))
+ fun s7 ⟨B7, o7⟩ => ?_
+ -- the blocks back
+ rw [WP.block_append_iff]
+ refine WP.mono (yperm_ok s7) fun s8 ⟨P8, o8⟩ => ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (yperm_ok s8) fun s9 ⟨P9, o9⟩ => ?_
+ have o59 := (o5.trans o6).trans (o7.trans (o8.trans o9))
+ have w0 : InRegions s9.wr (s9.gpr .rdx) 32 := by
+ rw [o59.wr, o59.gpr, g4.keep.2.2, g4.keep.gpr (by decide), o3.wr, o3.gpr, o12.wr, o12.gpr, hdx]
+ exact f_in32 hwf j0
+ have w1 : InRegions s9.wr (s9.gpr .rdx + BitVec.ofNat 64 32) 32 := by
+ rw [o59.wr, o59.gpr, g4.keep.2.2, g4.keep.gpr (by decide), o3.wr, o3.gpr, o12.wr, o12.gpr, hdx, a1]
+ exact f_in32 hwf j1
+ vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, State.setMem_ymm,
+ State.setMem_setMem, w0, w1, sx_ofNat (show 64 < 2 ^ 31 by decide)]
+ have g9 : s9.gpr = s4.gpr := o59.gpr
+ have m9 : s9.mem = s.mem := by rw [o59.mem, g4.mem, m3]
+ have dx9 : s9.gpr .rdx = coeffAddr fP (16 * m) := by
+ rw [g9, g4.keep.gpr (by decide), o3.gpr, o12.gpr, hdx]
+ rw [dx9, a1, m9]
+ have hR : ∀ i < 256, (layF blk F 4 zi (2 * (m + 1)))[i]! = if 16 * m ≤ i ∧ i < 16 * m + 16 then
+ (if i % (2 * 4) < 4 then (op P[i]! P[i + 4]! (zetas (zi (i / (2 * 4))))).1
+ else (op P[i - 4]! P[i]! (zetas (zi (i / (2 * 4))))).2) else P[i]! := fun i hi => by
+ have hF : ∀ j, 16 * m ≤ j → j < 256 → P[j]! = F[j]! := fun j h1 h2 => by
+ rw [← hP, layF_get hblk F (by decide) zi (by omega) h2, ite_eq_right (by omega)]
+ rw [layF_get hblk F (by decide) zi (by omega) hi]
+ by_cases h1 : 16 * m ≤ i ∧ i < 16 * m + 16
+ · rw [ite_eq_left (by omega), ite_eq_left h1]
+ by_cases h2 : i % (2 * 4) < 4
+ · rw [ite_eq_left h2, ite_eq_left h2, hF _ h1.1 hi, hF _ (by omega) (by omega)]
+ · rw [ite_eq_right h2, ite_eq_right h2, hF _ (by omega) (by omega), hF _ h1.1 hi]
+ · rw [ite_eq_right h1, ← hP, layF_get hblk F (by decide) zi (by omega) hi]
+ by_cases h3 : i < 16 * m
+ · rw [ite_eq_left (show i < 2 * 4 * (2 * (m + 1)) by omega),
+ ite_eq_left (show i < 2 * 4 * (2 * m) by omega)]
+ · rw [ite_eq_right (show ¬ i < 2 * 4 * (2 * (m + 1)) by omega),
+ ite_eq_right (show ¬ i < 2 * 4 * (2 * m) by omega)]
+ refine ⟨?_, ?_, ?_, ?_, ?_, ⟨⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, ?_⟩⟩
+ · refine polyIs_write2L (s := s9) hS j0 j1 (by omega) (fun l hl => ?_) (fun l hl => ?_)
+ (fun i hi h1 h2 => by rw [hR i hi, ite_eq_right (by omega)])
+ · rw [o9.lane .xmm4 (by decide) l hl, P8 l hl, perm20 _ _ hl]
+ rcases lane01 hl with rfl | rfl
+ · rw [ifp rfl]
+ exact (B7 0 (by decide)).1.congr fun e he => by
+ rw [hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)]
+ exact congrArg Prod.fst (op_idx op P zi (by omega) (by omega) (by omega))
+ · rw [ifn (by decide)]
+ exact (B7 0 (by decide)).2.congr fun e he => by
+ rw [hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)]
+ exact congrArg Prod.snd (op_idx op P zi (by omega) (by omega) (by omega))
+ · rw [P9 l hl, perm31 _ _ hl, o8.lane .xmm0 (by decide) 1 (by decide),
+ o8.lane .xmm3 (by decide) 1 (by decide)]
+ rcases lane01 hl with rfl | rfl
+ · rw [ifp rfl]
+ exact (B7 1 (by decide)).1.congr fun e he => by
+ rw [hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)]
+ exact congrArg Prod.fst (op_idx op P zi (by omega) (by omega) (by omega))
+ · rw [ifn (by decide)]
+ exact (B7 1 (by decide)).2.congr fun e he => by
+ rw [hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)]
+ exact congrArg Prod.snd (op_idx op P zi (by omega) (by omega) (by omega))
+ · rw [show BitVec.signExtend 64 (64 : BitVec 32) = BitVec.ofNat 64 (4 * 16) by decide, coeffAddr_add,
+ Nat.mul_succ]
+ · rw [g9, h84, o3.gpr, o12.gpr]
+ · rw [g9, g4.keep.gpr (by decide), o3.gpr, o12.gpr]
+ · rw [g9, g4.keep.gpr (by decide), o3.gpr, o12.gpr]
+ · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr
+ simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false, State.setMem_gpr]
+ rw [g9, g4.keep.gpr (by simp [hr]), o3.gpr, o12.gpr]
+ · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]
+ rw [o59.rd, g4.keep.2.1, o3.rd, o12.rd]
+ · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]
+ rw [o59.wr, g4.keep.2.2, o3.wr, o12.wr]
+ · exact frame_write2Y (Frame.refl _ _) j0 j1 _ _
+ · exact ylanes_gpr (s := s9) (fun r l => by simp only [lane_setReg, lane_setFlags, State.setMem_lane])
+ o59 (ylanes_gpr (s := s3) l4 (o12.trans o3) hc (by decide) (by decide)) (by decide) (by decide)
+ · exact o59.mxcsr.trans (g4.mxcsr.trans (o12.trans o3).mxcsr)
+
+theorem ylay4_ok {fP sP : Addr} (k : Nat) (o₀ o₁ : BitVec 8) (dz : BitVec 32) (zi kb : Nat → Nat)
+ (hkb0 : kb 0 = k) (hk : ∀ m < 16, kb m + 4 ≤ 256)
+ (hsel : ∀ m < 16, ∀ e < 4, kb m + sel o₀ e = zi (2 * m) ∧ kb m + sel o₁ e = zi (2 * m + 1))
+ (hstep : ∀ m < 16, coeffAddr sP (kb m) + BitVec.signExtend 64 dz = coeffAddr sP (kb (m + 1)))
+ {F : Poly} {s : State} (hc : YConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP)
+ (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr)
+ (hd : (pR sP).Disjoint (pR fP)) :
+ WP isa (ylay4 bf k o₀ o₁ dz) s fun s' => PolyIs s'.mem fP (layF blk F 4 zi 32) ∧ BInvY fP s s' := by
+ have hk0 : k + 4 ≤ 256 := by have := hk 0 (by decide); omega
+ refine WP.seq (WP.mono (ypre21 k hk0 hdi hsi) fun w ⟨hdx, h8, og, hy⟩ => ?_)
+ have cw : YConsts w := ylanes_gpr (s := s) (og.lane hy) (YOnly.refl [] s) hc (by decide) (by decide)
+ refine WP.mono (wp_rcxLoopY (N := 16) (by decide) (by decide)
+ (fun i u => PolyIs u.mem fP (layF blk F 4 zi (2 * i)) ∧ u.gpr .rdx = coeffAddr fP (16 * i) ∧
+ u.gpr .r8 = coeffAddr sP (kb i) ∧ BInvY fP w u)
+ (fun u ou hu _ => ⟨by rw [ou.mem, og.mem]; exact hS,
+ by rw [ou.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero],
+ by rw [ou.keep.gpr (by decide), h8, hkb0], ⟨ou.keep.mono (by simp), by rw [ou.mem]; exact Frame.refl _ _,
+ ylanes_gpr (s := w) (ou.lane hu) (YOnly.refl [] w) cw (by decide) (by decide), ou.mxcsr⟩⟩)
+ (fun i hi u ⟨hS', hdx', h8', hb'⟩ => ?_)) fun u ⟨hS', _, _, hb'⟩ =>
+ ⟨hS', ⟨(og.keep.trans hb'.keep).mono (by simp), by rw [← og.mem]; exact hb'.frame, hb'.consts,
+ by rw [hb'.mxcsr, og.mxcsr]⟩⟩
+ have hT' : Tab zmTab u.mem sP 256 := (by rw [og.mem]; exact hT : Tab zmTab w.mem sP 256).frame hb'.frame
+ (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide)
+ have hwf' : pR fP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hwf
+ have hw' : pR sP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hw
+ rw [show [Instr.vmovdquLoad .l256 .xmm4 (at_ .rdx 0), .vmovdquLoad .l256 .xmm5 (at_ .rdx 32)] ++ yzetaS o₀ o₁ ++
+ [.alu .add .r8 (.imm dz), .vop (.vperm2i128 .xmm0 .xmm4 .xmm5 0x20),
+ .vop (.vperm2i128 .xmm1 .xmm4 .xmm5 0x31)] ++ toY bf ++
+ [.vop (.vperm2i128 .xmm4 .xmm0 .xmm3 0x20), .vop (.vperm2i128 .xmm5 .xmm0 .xmm3 0x31),
+ .vmovdquStore .l256 (at_ .rdx 0) .xmm4, .vmovdquStore .l256 (at_ .rdx 32) .xmm5,
+ .alu .add .rdx (.imm 64)] ++ [.alu .sub .rcx (.imm 1)] = ybody4 bf o₀ o₁ dz by
+ simp [List.append_assoc]]
+ exact WP.mono (ystep4 hbf hY hblk hi o₀ o₁ dz zi (hk i hi) (hsel i hi) hb'.consts hdx' h8' hS' hT' hwf' hw')
+ fun u' ⟨hS'', hdx'', h8'', hcx, hzf, hb''⟩ => ⟨⟨hS'', hdx'', by rw [h8'', h8', hstep i hi],
+ hb'.trans hb''⟩, hcx, hzf⟩
+
+end
+
+end VG.Proof.MlDsa.X86_64.Arith
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay21.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay21.lean
new file mode 100644
index 000000000..f8bf6a6bd
--- /dev/null
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay21.lean
@@ -0,0 +1,526 @@
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YLay
+
+/-!
+# ML-DSA on x86-64: the layers of the NTT and its inverse with `len` = 2 and 1 on AVX2 registers
+
+Untrusted: everything here is checked by Lean. Each iteration of these
+layers loads sixteen coefficients, from `j`, into `ymm0` and `ymm1` (or
+`ymm2`), and in each lane `l` runs `vlay2`'s or `vlay1`'s gathering,
+butterflies and interleaving back (`Ntt.lean`) on the four coefficients
+from `j + 4l` and the four from `j + 8 + 4l` (`core2_ok`, `core1_ok`), with
+the zetas of their blocks in lane `l` of `ymm13` (`yzetaS_ok`, `yzeta8_ok`,
+`yzeta8R_ok`); `ystep21` is an iteration for any such code, and `ylay2_ok`
+and `ylay1_ok` the layers.
+-/
+
+namespace VG.Proof.MlDsa.X86_64.Arith
+
+open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith
+open VG.Proof.MlDsa.Arith
+open VG.Proof.MlKem.X86_64 (Keep XOnly XKeep YOnly ylanes yld_ok ifp ifn sel sel_lt add_ofNat_zero GOnly
+ addR_ok wp_rcxLoopY wp_cons_iff lane_setReg lane_setFlags State.setMem_ymm State.setMem_setMem q256lo q256hi
+ lo4 hi4)
+open VG.Impl.MlKem.X86_64 (xb xmov toY rcxLoop)
+open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt PolyIs zetas)
+
+/-! ## An iteration -/
+
+/-- The body of the loop of the layers with `len` = 2 and 1. -/
+abbrev ybody21 (r2 : XReg) (zl core : List Instr) (dz : BitVec 32) : List Instr :=
+ [.vmovdquLoad .l256 .xmm0 (at_ .rdx 0)] ++ ([.vmovdquLoad .l256 r2 (at_ .rdx 32)] ++ (zl ++
+ ([.alu .add .r8 (.imm dz)] ++ (toY core ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1, .alu .add .rdx (.imm 64),
+ .alu .sub .rcx (.imm 1)]))))
+
+/-- An iteration of a layer with `len` = 2 or 1: the sixteen coefficients of
+`G` from `j`, in lane `l` the four from `j + 4l` and the four from
+`j + 8 + 4l`, become those of `R`, with the zetas `ζ l` that `zl` leaves in
+the lanes of `ymm13`. -/
+theorem ystep21 {core zl : List Instr} {r2 : XReg} {zs : List XReg} {dz : BitVec 32}
+ (hY : laneSseBlock (toY core) = some core) (h0 : XReg.xmm0 ∉ zs) (h2 : r2 ∉ zs)
+ (h14 : XReg.xmm14 ∉ [XReg.xmm0] ++ [r2] ++ zs) (h15 : XReg.xmm15 ∉ [XReg.xmm0] ++ [r2] ++ zs)
+ (h20 : XReg.xmm0 ∉ [r2]) {fP : Addr} {j : Nat} (hj : j + 16 ≤ 256) {G R : Poly}
+ {ζ : Nat → Nat → Zq} {s : State} (hc : YConsts s) (hdx : s.gpr .rdx = coeffAddr fP j)
+ (hS : PolyIs s.mem fP G) (hw : pR fP ∈ s.wr)
+ (hz : ∀ s', XKeep s s' →
+ WP isa (.block zl) s' fun s'' => (∀ l < 2, ZLanes (s''.lane .xmm13 l) (ζ l) ∧
+ ZOdd (s''.lane .xmm13 l) (s''.lane .xmm12 l)) ∧ YOnly zs s' s'')
+ (hcore : ∀ l < 2, ∀ t : State, VConsts t → DLanes (t.xmm .xmm0) (fun e => G[j + 4 * l + e]!) →
+ DLanes (t.xmm r2) (fun e => G[j + 8 + 4 * l + e]!) → ZLanes (t.xmm .xmm13) (ζ l) →
+ ZOdd (t.xmm .xmm13) (t.xmm .xmm12) →
+ WP isa (.block core) t fun t' => (DLanes (t'.xmm .xmm0) (fun e => R[j + 4 * l + e]!) ∧
+ DLanes (t'.xmm .xmm1) (fun e => R[j + 8 + 4 * l + e]!)) ∧
+ XOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm4] t t')
+ (hR : ∀ i < 256, i < j ∨ j + 16 ≤ i → R[i]! = G[i]!) :
+ WP isa (.block (ybody21 r2 zl core dz)) s fun s' =>
+ PolyIs s'.mem fP R ∧ s'.gpr .rdx = coeffAddr fP (j + 16) ∧
+ s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧
+ s'.zf = some (s.gpr .rcx - 1 == 0) ∧ BInvY fP s s' := by
+ have j0 : j + 8 ≤ 256 := by omega
+ have j1 : j + 8 + 8 ≤ 256 := by omega
+ have a1 : coeffAddr fP j + BitVec.ofNat 64 32 = coeffAddr fP (j + 8) := coeffAddr_add _ _ 8
+ have r0 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 0) 32 := by
+ rw [hdx, add_ofNat_zero]; exact f_in32 (List.mem_append_right s.rd hw) j0
+ have r1 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 32) 32 := by
+ rw [hdx, a1]; exact f_in32 (List.mem_append_right s.rd hw) j1
+ rw [ybody21, WP.block_append_iff]
+ refine WP.mono (yld_ok r0) fun s1 ⟨L0, o1⟩ => ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (yld_ok (d := r2) (by rw [o1.rd, o1.wr, o1.gpr]; exact r1)) fun s2 ⟨L2, o2⟩ => ?_
+ have o12 := o1.trans o2
+ rw [WP.block_append_iff]
+ refine WP.mono (hz s2 o12.toXKeep) fun s3 ⟨Z3, o3⟩ => ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (addR_ok .r8 dz s3) fun s4 ⟨h84, g4, y4⟩ => ?_
+ have l4 := g4.lane y4
+ have c4 : YConsts s4 := ylanes_gpr (s := s3) l4 (o12.trans o3) hc h14 h15
+ rw [WP.block_append_iff]
+ refine WP.mono (ylanes hY (P := fun l t => DLanes (t.xmm .xmm0) (fun e => R[j + 4 * l + e]!) ∧
+ DLanes (t.xmm .xmm1) (fun e => R[j + 8 + 4 * l + e]!))
+ fun l hl => hcore l hl _ (c4 l hl)
+ (by rw [State.proj_xmm, l4, o3.lane _ h0 l hl, o2.lane _ h20 l hl, L0 l hl, hdx, add_ofNat_zero]
+ exact dlanes_loadY hS j0 hl)
+ (by rw [State.proj_xmm, l4, o3.lane _ h2 l hl, L2 l hl, o1.gpr, o1.mem, hdx, a1]
+ exact dlanes_loadY hS j1 hl)
+ (by rw [State.proj_xmm, l4]; exact (Z3 l hl).1)
+ (by rw [State.proj_xmm, State.proj_xmm, l4, l4]; exact (Z3 l hl).2)) fun s5 ⟨C5, o5⟩ => ?_
+ have m5 : s5.mem = s.mem := by rw [o5.mem, g4.mem, o3.mem, o12.mem]
+ have g5 : s5.gpr = s4.gpr := o5.gpr
+ have dx5 : s5.gpr .rdx = coeffAddr fP j := by rw [g5, g4.keep.gpr (by decide), o3.gpr, o12.gpr, hdx]
+ have k5 : s5.rd = s.rd ∧ s5.wr = s.wr := ⟨by rw [o5.rd, g4.keep.2.1, o3.rd, o12.rd],
+ by rw [o5.wr, g4.keep.2.2, o3.wr, o12.wr]⟩
+ have w0 : InRegions s5.wr (s5.gpr .rdx) 32 := by rw [k5.2, dx5]; exact f_in32 hw j0
+ have w1 : InRegions s5.wr (s5.gpr .rdx + BitVec.ofNat 64 32) 32 := by rw [k5.2, dx5, a1]; exact f_in32 hw j1
+ vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd,
+ State.setMem_ymm, State.setMem_setMem, w0, w1, sx_ofNat (show 64 < 2 ^ 31 by decide)]
+ rw [dx5, a1, m5]
+ refine ⟨?_, ?_, ?_, ?_, ?_, ⟨⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, ?_⟩⟩
+ · exact polyIs_write2L (s := s5) hS j0 j1 (by omega) (fun l hl => (C5 l hl).1)
+ (fun l hl => (C5 l hl).2) fun i hi _ _ => hR i hi (by omega)
+ · rw [show BitVec.signExtend 64 (64 : BitVec 32) = BitVec.ofNat 64 (4 * 16) by decide, coeffAddr_add]
+ · rw [g5, h84, o3.gpr, o12.gpr]
+ · rw [g5, g4.keep.gpr (by decide), o3.gpr, o12.gpr]
+ · rw [g5, g4.keep.gpr (by decide), o3.gpr, o12.gpr]
+ · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr
+ simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false, State.setMem_gpr]
+ rw [g5, g4.keep.gpr (by simp [hr]), o3.gpr, o12.gpr]
+ · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; exact k5.1
+ · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; exact k5.2
+ · exact frame_write2Y (Frame.refl _ _) j0 j1 _ _
+ · exact ylanes_gpr (s := s5) (fun r l => by simp only [lane_setReg, lane_setFlags, State.setMem_lane]) o5 c4
+ (by decide) (by decide)
+ · exact o5.mxcsr.trans (g4.mxcsr.trans (o12.trans o3).mxcsr)
+
+/-! ## The gatherings, the butterflies and the interleavings back of a lane -/
+
+theorem gath2_ok (t : State) :
+ WP isa (.block gath2) t fun t' => (t'.xmm .xmm0 = XBinOp.eval .punpcklqdq (t.xmm .xmm0) (t.xmm .xmm1) ∧
+ t'.xmm .xmm1 = XBinOp.eval .punpckhqdq (t.xmm .xmm0) (t.xmm .xmm1)) ∧ XOnly [.xmm2, .xmm0, .xmm1] t t' := by
+ simp only [gath2, xmov, xb]
+ vrun [eval_movdqa]
+ exact ⟨by first | trivial | simp, by xonly⟩
+
+theorem scat2_ok (t : State) :
+ WP isa (.block scat2) t fun t' => (t'.xmm .xmm0 = XBinOp.eval .punpcklqdq (t.xmm .xmm0) (t.xmm .xmm3) ∧
+ t'.xmm .xmm1 = XBinOp.eval .punpckhqdq (t.xmm .xmm0) (t.xmm .xmm3)) ∧ XOnly [.xmm1, .xmm0] t t' := by
+ simp only [scat2, xmov, xb]
+ vrun [eval_movdqa]
+ exact ⟨by first | trivial | simp, by xonly⟩
+
+theorem gath1_ok (t : State) :
+ WP isa (.block gath1) t fun t' =>
+ (t'.xmm .xmm0 = XBinOp.eval .punpcklqdq (shufDwords (t.xmm .xmm0) 0xD8) (shufDwords (t.xmm .xmm2) 0xD8) ∧
+ t'.xmm .xmm1 = XBinOp.eval .punpckhqdq (shufDwords (t.xmm .xmm0) 0xD8) (shufDwords (t.xmm .xmm2) 0xD8)) ∧
+ XOnly [.xmm0, .xmm2, .xmm1] t t' := by
+ simp only [gath1, xmov, xb]
+ vrun [eval_movdqa]
+ exact ⟨by first | trivial | simp, by xonly⟩
+
+theorem scat1_ok (t : State) :
+ WP isa (.block scat1) t fun t' => (t'.xmm .xmm0 = XBinOp.eval .punpckldq (t.xmm .xmm0) (t.xmm .xmm3) ∧
+ t'.xmm .xmm1 = XBinOp.eval .punpckhdq (t.xmm .xmm0) (t.xmm .xmm3)) ∧ XOnly [.xmm1, .xmm0] t t' := by
+ simp only [scat1, xmov, xb]
+ vrun [eval_movdqa]
+ exact ⟨by first | trivial | simp, by xonly⟩
+
+section
+variable {bf : List Instr} {op : Zq → Zq → Zq → Zq × Zq} (hbf : VBflyOk bf op)
+include hbf
+
+/-- `vlay2`'s work on the coefficients `A` of `xmm0` and `B` of `xmm1`: the
+blocks `A` and `B` of `len = 2`, with the zetas `ζ` (the first two for `A`,
+the last two for `B`). -/
+theorem core2_ok {t : State} (hc : VConsts t) {A B ζ : Nat → Zq} (hA : DLanes (t.xmm .xmm0) A)
+ (hB : DLanes (t.xmm .xmm1) B) (hz : ZLanes (t.xmm .xmm13) ζ) (ho : ZOdd (t.xmm .xmm13) (t.xmm .xmm12)) :
+ WP isa (.block (gath2 ++ bf ++ scat2)) t fun t' =>
+ (DLanes (t'.xmm .xmm0) (fun e => if e < 2 then (op (A e) (A (2 + e)) (ζ e)).1
+ else (op (A (e - 2)) (A e) (ζ (e - 2))).2) ∧
+ DLanes (t'.xmm .xmm1) (fun e => if e < 2 then (op (B e) (B (2 + e)) (ζ (2 + e))).1
+ else (op (B (e - 2)) (B e) (ζ e)).2)) ∧ XOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm4] t t' := by
+ rw [List.append_assoc, WP.block_append_iff]
+ refine WP.mono (gath2_ok t) fun t1 ⟨⟨e0, e1⟩, o1⟩ => ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (hbf _ (xonly_vconsts o1 hc (by decide) (by decide))
+ (fun e => if e < 2 then A e else B (e - 2)) (fun e => if e < 2 then A (2 + e) else B e) ζ
+ (fun e he => by
+ dsimp only; rw [e0, dword_punpcklqdq _ _ he]
+ by_cases h : e < 2 <;> simp only [h, ite_true, ite_false]
+ · exact hA e he
+ · exact hB (e - 2) (by omega))
+ (fun e he => by
+ dsimp only; rw [e1, dword_punpckhqdq _ _ he]
+ by_cases h : e < 2 <;> simp only [h, ite_true, ite_false]
+ · exact hA (2 + e) (by omega)
+ · exact hB e he)
+ (by rw [o1.xmm _ (by decide)]; exact hz) (by rw [o1.xmm _ (by decide), o1.xmm _ (by decide)]; exact ho))
+ fun t2 ⟨X, Y, o2⟩ => ?_
+ refine WP.mono (scat2_ok t2) fun t3 ⟨⟨f0, f1⟩, o3⟩ => ⟨⟨fun e he => ?_, fun e he => ?_⟩, ?_⟩
+ · rw [f0, dword_punpcklqdq _ _ he]
+ split
+ · rw [X e he]; dsimp only
+ rw [ite_eq_left_of_eq_true _ _ (eq_true ‹e < 2›), ite_eq_left_of_eq_true _ _ (eq_true ‹e < 2›),
+ ite_eq_left_of_eq_true _ _ (eq_true ‹e < 2›)]
+ · rw [Y (e - 2) (by omega)]; dsimp only
+ rw [ite_eq_left_of_eq_true _ _ (eq_true (show e - 2 < 2 by omega)),
+ ite_eq_left_of_eq_true _ _ (eq_true (show e - 2 < 2 by omega)),
+ ite_eq_right_of_eq_false _ _ (eq_false ‹¬ e < 2›), show 2 + (e - 2) = e by omega]
+ · rw [f1, dword_punpckhqdq _ _ he]
+ split
+ · rw [X (2 + e) (by omega)]; dsimp only
+ rw [ite_eq_right_of_eq_false _ _ (eq_false (show ¬ 2 + e < 2 by omega)),
+ ite_eq_right_of_eq_false _ _ (eq_false (show ¬ 2 + e < 2 by omega)),
+ ite_eq_left_of_eq_true _ _ (eq_true ‹e < 2›), show 2 + e - 2 = e by omega]
+ · rw [Y e he]; dsimp only
+ rw [ite_eq_right_of_eq_false _ _ (eq_false ‹¬ e < 2›), ite_eq_right_of_eq_false _ _ (eq_false ‹¬ e < 2›),
+ ite_eq_right_of_eq_false _ _ (eq_false ‹¬ e < 2›)]
+ · exact ((o1.trans o2).trans o3).mono (by simp)
+
+/-- `vlay1`'s work on the coefficients `A` of `xmm0` and `B` of `xmm2`: the
+blocks `A₀₁`, `A₂₃`, `B₀₁` and `B₂₃` of `len = 1`, with the zetas `ζ`. -/
+theorem core1_ok {t : State} (hc : VConsts t) {A B ζ : Nat → Zq} (hA : DLanes (t.xmm .xmm0) A)
+ (hB : DLanes (t.xmm .xmm2) B) (hz : ZLanes (t.xmm .xmm13) ζ) (ho : ZOdd (t.xmm .xmm13) (t.xmm .xmm12)) :
+ WP isa (.block (gath1 ++ bf ++ scat1)) t fun t' =>
+ (DLanes (t'.xmm .xmm0) (fun e => if e % 2 = 0 then (op (A e) (A (e + 1)) (ζ (e / 2))).1
+ else (op (A (e - 1)) (A e) (ζ (e / 2))).2) ∧
+ DLanes (t'.xmm .xmm1) (fun e => if e % 2 = 0 then (op (B e) (B (e + 1)) (ζ (2 + e / 2))).1
+ else (op (B (e - 1)) (B e) (ζ (2 + e / 2))).2)) ∧ XOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm4] t t' := by
+ rw [List.append_assoc, WP.block_append_iff]
+ refine WP.mono (gath1_ok t) fun t1 ⟨⟨e0, e1⟩, o1⟩ => ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (hbf _ (xonly_vconsts o1 hc (by decide) (by decide))
+ (fun e => if e < 2 then A (2 * e) else B (2 * (e - 2))) (fun e => if e < 2 then A (2 * e + 1) else B (2 * (e - 2) + 1))
+ ζ
+ (fun e he => by
+ dsimp only; rw [e0, dword_punpcklqdq _ _ he]
+ by_cases h : e < 2 <;> simp only [h, ite_true, ite_false]
+ · rw [dword_d8 _ he, ite_eq_left h]; exact hA _ (by omega)
+ · rw [dword_d8 _ (by omega), ite_eq_left (by omega)]; exact hB _ (by omega))
+ (fun e he => by
+ dsimp only; rw [e1, dword_punpckhqdq _ _ he]
+ by_cases h : e < 2 <;> simp only [h, ite_true, ite_false]
+ · rw [dword_d8 _ (by omega), ite_eq_right (by omega), show 2 * (2 + e - 2) + 1 = 2 * e + 1 by omega]
+ exact hA _ (by omega)
+ · rw [dword_d8 _ he, ite_eq_right h]; exact hB _ (by omega))
+ (by rw [o1.xmm _ (by decide)]; exact hz) (by rw [o1.xmm _ (by decide), o1.xmm _ (by decide)]; exact ho))
+ fun t2 ⟨X, Y, o2⟩ => ?_
+ refine WP.mono (scat1_ok t2) fun t3 ⟨⟨f0, f1⟩, o3⟩ => ⟨⟨fun e he => ?_, fun e he => ?_⟩, ?_⟩
+ · rw [f0, dword_punpckldq' _ _ he]
+ split
+ · rw [X (e / 2) (by omega)]; dsimp only
+ rw [ite_eq_left (by omega), ite_eq_left (by omega), ite_eq_left ‹e % 2 = 0›,
+ show 2 * (e / 2) = e by omega]
+ · rw [Y (e / 2) (by omega)]; dsimp only
+ rw [ite_eq_left (by omega), ite_eq_left (by omega), ite_eq_right ‹¬ e % 2 = 0›,
+ show 2 * (e / 2) = e - 1 by omega, show e - 1 + 1 = e by omega]
+ · rw [f1, dword_punpckhdq' _ _ he]
+ split
+ · rw [X (2 + e / 2) (by omega)]; dsimp only
+ rw [ite_eq_right (by omega), ite_eq_right (by omega), ite_eq_left ‹e % 2 = 0›,
+ show 2 * (2 + e / 2 - 2) = e by omega]
+ · rw [Y (2 + e / 2) (by omega)]; dsimp only
+ rw [ite_eq_right (by omega), ite_eq_right (by omega), ite_eq_right ‹¬ e % 2 = 0›,
+ show 2 * (2 + e / 2 - 2) = e - 1 by omega, show e - 1 + 1 = e by omega]
+ · exact ((o1.trans o2).trans o3).mono (by simp)
+
+end
+
+/-! ## The zetas of the layer with `len = 1` -/
+
+/-- A doubleword of four of the zetas, at index `i` of the table. -/
+theorem dword_tab {m : Mem} {zP : Addr} (ht : Tab zmTab m zP 256) {j e i : Nat} (he : e < 4) (hi : j + e = i)
+ (hi' : i < 256) : (dword (m.readW (coeffAddr zP j) 128) e).toNat = (zetas i).val * 2 ^ 32 % q := by
+ subst hi; rw [dword_readW _ _ he, coeffAddr_add]; exact tab_zeta ht hi'
+
+/-- `vpermq` with `0x27`: lane 0 is `punpckhqdq` of the upper and the lower lane, lane 1 their `punpcklqdq`. -/
+theorem perm27 (a b : BitVec 128) :
+ permQwords (b ++ a) 0x27 = qword256 (b ++ a) 0 ++ qword256 (b ++ a) (2 + 0) ++
+ qword256 (b ++ a) 1 ++ qword256 (b ++ a) (2 + 1) := rfl
+
+theorem perm27_lo (a b : BitVec 128) :
+ (permQwords (b ++ a) 0x27).extractLsb' 0 128 = XBinOp.eval .punpckhqdq b a := by
+ rw [perm27, q256hi, q256hi, q256lo _ _ (by decide), q256lo _ _ (by decide), lo4]; rfl
+
+theorem perm27_hi (a b : BitVec 128) :
+ (permQwords (b ++ a) 0x27).extractLsb' 128 128 = XBinOp.eval .punpcklqdq b a := by
+ rw [perm27, q256hi, q256hi, q256lo _ _ (by decide), q256lo _ _ (by decide), hi4]; rfl
+
+/-- `vpermq d, r, 0x27`. -/
+theorem ypermq27_ok {d r : XReg} (s : State) :
+ WP isa (.block [.vop (.vpermq d r 0x27)]) s fun s' =>
+ s'.lane d 0 = XBinOp.eval .punpckhqdq (s.lane r 1) (s.lane r 0) ∧
+ s'.lane d 1 = XBinOp.eval .punpcklqdq (s.lane r 1) (s.lane r 0) ∧ YOnly [d] s s' := by
+ apply WP.of_runBlock
+ simp only [runBlock_cons, runStep_some, runBlock_nil, exec, VOp.exec, Option.some.injEq, exists_eq_left']
+ refine ⟨?_, ?_, ⟨⟨rfl, rfl, rfl, rfl, rfl⟩, fun r' hr l hl => ?_⟩⟩
+ · rw [State.lane_setV256, ifp rfl, ifp rfl, State.ymm_eq, perm27_lo]
+ · rw [State.lane_setV256, ifp rfl, ifn (by decide), State.ymm_eq, perm27_hi]
+ · rw [State.lane_setV256, ifn (by simpa using hr)]
+
+/-- The eight zetas at index `k` of the table, in the order of the blocks of
+`ylay1` for `NTT`: in lane `l`, zetas `k + 2l`, `k + 2l + 1`, `k + 2l + 4`
+and `k + 2l + 5`. -/
+theorem yzeta8_ok {zP : Addr} {k : Nat} (hk : k + 8 ≤ 256) {s : State} (h8 : s.gpr .r8 = coeffAddr zP k)
+ (hin : InRegions (s.rd ++ s.wr) (coeffAddr zP k) 32) (ht : Tab zmTab s.mem zP 256) :
+ WP isa (.block yzeta8) s fun s' =>
+ (∀ l < 2, ZLanes (s'.lane .xmm13 l) (fun e => zetas (k + (2 * l + e + 2 * (e / 2)))) ∧
+ ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧ YOnly [.xmm13, .xmm12] s s' := by
+ rw [yzeta8, WP.block_append_iff, wp_cons_iff]
+ refine WP.mono (yld_ok (by rw [h8, add_ofNat_zero]; exact hin)) fun s1 ⟨L1, o1⟩ => ?_
+ refine WP.mono (VG.Proof.MlKem.X86_64.ypermq_ok s1) fun s2 ⟨e0, e1, o2⟩ => ?_
+ refine WP.mono (yF5_ok s2) fun s3 ⟨f3, o3⟩ => ⟨fun l hl => ?_, ((o1.trans o2).trans o3).mono (by simp)⟩
+ rw [f3 l hl, o3.lane _ (by decide) l hl]
+ refine ⟨fun e he => ?_, zodd_F5 _⟩
+ have a : ∀ l < 2, s1.lane .xmm13 l = s.mem.readW (coeffAddr zP (k + 4 * l)) 128 := fun l hl => by
+ rw [L1 l hl, h8, add_ofNat_zero, lane_load]
+ rcases lane01 hl with rfl | rfl
+ · rw [e0, dword_punpcklqdq _ _ he, a 0 (by decide), a 1 (by decide)]
+ split
+ · exact dword_tab ht he (by omega) (by omega)
+ · exact dword_tab ht (by omega) (by omega) (by omega)
+ · rw [e1, dword_punpckhqdq _ _ he, a 0 (by decide), a 1 (by decide)]
+ split
+ · exact dword_tab ht (by omega) (by omega) (by omega)
+ · exact dword_tab ht he (by omega) (by omega)
+
+theorem dword_b1 (x : BitVec 128) {e : Nat} (he : e < 4) :
+ dword (shufDwords x 0xB1) e = dword x (if e % 2 = 0 then e + 1 else e - 1) := by
+ rw [dword_shufDwords _ _ he]
+ rcases cases4 he with rfl | rfl | rfl | rfl <;> rfl
+
+theorem zsseR_ok (t : State) :
+ WP isa (.block [.xop (.pshufd .xmm13 .xmm13 0xB1), .xop (.pshufd .xmm12 .xmm13 0xF5)]) t fun t' =>
+ (t'.xmm .xmm13 = shufDwords (t.xmm .xmm13) 0xB1 ∧
+ t'.xmm .xmm12 = shufDwords (shufDwords (t.xmm .xmm13) 0xB1) 0xF5) ∧ XOnly [.xmm13, .xmm12] t t' := by
+ vrun
+ exact ⟨by first | trivial | simp, by xonly⟩
+
+/-- The eight zetas at index `k` of the table, in the order of the blocks of
+`ylay1` for `NTT⁻¹`, which take them in decreasing order: in lane `l`, zetas
+`k + 7 - 2l`, `k + 6 - 2l`, `k + 3 - 2l` and `k + 2 - 2l`. -/
+theorem yzeta8R_ok {zP : Addr} {k : Nat} (hk : k + 8 ≤ 256) {s : State} (h8 : s.gpr .r8 = coeffAddr zP k)
+ (hin : InRegions (s.rd ++ s.wr) (coeffAddr zP k) 32) (ht : Tab zmTab s.mem zP 256) :
+ WP isa (.block yzeta8R) s fun s' =>
+ (∀ l < 2, ZLanes (s'.lane .xmm13 l) (fun e => zetas (k + 7 - (2 * l + e + 2 * (e / 2)))) ∧
+ ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧ YOnly [.xmm13, .xmm12] s s' := by
+ rw [yzeta8R, WP.block_append_iff, wp_cons_iff]
+ refine WP.mono (yld_ok (by rw [h8, add_ofNat_zero]; exact hin)) fun s1 ⟨L1, o1⟩ => ?_
+ refine WP.mono (ypermq27_ok s1) fun s2 ⟨e0, e1, o2⟩ => ?_
+ refine WP.mono (ylanes (by decide) (P := fun l t => t.xmm .xmm13 = shufDwords (s2.lane .xmm13 l) 0xB1 ∧
+ t.xmm .xmm12 = shufDwords (shufDwords (s2.lane .xmm13 l) 0xB1) 0xF5)
+ fun l _ => zsseR_ok (s2.proj l)) fun s3 ⟨l3, o3⟩ => ⟨fun l hl => ?_, ((o1.trans o2).trans o3).mono (by simp)⟩
+ have e13 : s3.lane .xmm13 l = _ := (l3 l hl).1
+ have e12 : s3.lane .xmm12 l = _ := (l3 l hl).2
+ rw [e12, e13]
+ refine ⟨fun e he => ?_, zodd_F5 _⟩
+ have a : ∀ l < 2, s1.lane .xmm13 l = s.mem.readW (coeffAddr zP (k + 4 * l)) 128 := fun l hl => by
+ rw [L1 l hl, h8, add_ofNat_zero, lane_load]
+ have he' : (if e % 2 = 0 then e + 1 else e - 1) < 4 := by split <;> omega
+ rw [dword_b1 _ he]
+ generalize hf : (if e % 2 = 0 then e + 1 else e - 1) = f at he'
+ have hf' : f = if e % 2 = 0 then e + 1 else e - 1 := hf.symm
+ rcases lane01 hl with rfl | rfl
+ · rw [e0, dword_punpckhqdq _ _ he', a 0 (by decide), a 1 (by decide)]
+ split
+ · exact dword_tab ht (by omega) (by split at hf' <;> omega) (by omega)
+ · exact dword_tab ht he' (by split at hf' <;> omega) (by omega)
+ · rw [e1, dword_punpcklqdq _ _ he', a 0 (by decide), a 1 (by decide)]
+ split
+ · exact dword_tab ht he' (by split at hf' <;> omega) (by omega)
+ · exact dword_tab ht (by omega) (by split at hf' <;> omega) (by omega)
+
+/-! ## The layers -/
+
+section
+variable {op : Zq → Zq → Zq → Zq × Zq} {blk : Poly → Nat → Nat → Nat → Nat → Poly} (hblk : BlkOk blk op)
+include hblk
+
+/-- The sixteen coefficients from `16i` after the next blocks of the layer
+with `len = 2`. -/
+theorem layF2_next (F : Poly) (zi : Nat → Nat) {i : Nat} (hi : i < 16) {x : Nat} (hx : x < 256) :
+ (layF blk F 2 zi (4 * (i + 1)))[x]! = if 16 * i ≤ x ∧ x < 16 * i + 16 then
+ (if x % (2 * 2) < 2 then
+ (op (layF blk F 2 zi (4 * i))[x]! (layF blk F 2 zi (4 * i))[x + 2]! (zetas (zi (x / (2 * 2))))).1
+ else (op (layF blk F 2 zi (4 * i))[x - 2]! (layF blk F 2 zi (4 * i))[x]! (zetas (zi (x / (2 * 2))))).2)
+ else (layF blk F 2 zi (4 * i))[x]! := by
+ have hF : ∀ y, 16 * i ≤ y → y < 256 → (layF blk F 2 zi (4 * i))[y]! = F[y]! := fun y h1 h2 => by
+ rw [layF_get hblk F (by decide) zi (by omega) h2, ite_eq_right (by omega)]
+ rw [layF_get hblk F (by decide) zi (by omega) hx]
+ by_cases h1 : 16 * i ≤ x ∧ x < 16 * i + 16
+ · rw [ite_eq_left (by omega), ite_eq_left h1]
+ by_cases h2 : x % (2 * 2) < 2
+ · rw [ite_eq_left h2, ite_eq_left h2, hF _ h1.1 hx, hF _ (by omega) (by omega)]
+ · rw [ite_eq_right h2, ite_eq_right h2, hF _ (by omega) (by omega), hF _ h1.1 hx]
+ · rw [ite_eq_right h1, layF_get hblk F (by decide) zi (by omega) hx]
+ by_cases h3 : x < 16 * i
+ · rw [ite_eq_left (show x < 2 * 2 * (4 * (i + 1)) by omega), ite_eq_left (show x < 2 * 2 * (4 * i) by omega)]
+ · rw [ite_eq_right (show ¬ x < 2 * 2 * (4 * (i + 1)) by omega),
+ ite_eq_right (show ¬ x < 2 * 2 * (4 * i) by omega)]
+
+/-- The sixteen coefficients from `16i` after the next blocks of the layer
+with `len = 1`. -/
+theorem layF1_next (F : Poly) (zi : Nat → Nat) {i : Nat} (hi : i < 16) {x : Nat} (hx : x < 256) :
+ (layF blk F 1 zi (8 * (i + 1)))[x]! = if 16 * i ≤ x ∧ x < 16 * i + 16 then
+ (if x % (2 * 1) < 1 then
+ (op (layF blk F 1 zi (8 * i))[x]! (layF blk F 1 zi (8 * i))[x + 1]! (zetas (zi (x / (2 * 1))))).1
+ else (op (layF blk F 1 zi (8 * i))[x - 1]! (layF blk F 1 zi (8 * i))[x]! (zetas (zi (x / (2 * 1))))).2)
+ else (layF blk F 1 zi (8 * i))[x]! := by
+ have hF : ∀ y, 16 * i ≤ y → y < 256 → (layF blk F 1 zi (8 * i))[y]! = F[y]! := fun y h1 h2 => by
+ rw [layF_get hblk F (by decide) zi (by omega) h2, ite_eq_right (by omega)]
+ rw [layF_get hblk F (by decide) zi (by omega) hx]
+ by_cases h1 : 16 * i ≤ x ∧ x < 16 * i + 16
+ · rw [ite_eq_left (by omega), ite_eq_left h1]
+ by_cases h2 : x % (2 * 1) < 1
+ · rw [ite_eq_left h2, ite_eq_left h2, hF _ h1.1 hx, hF _ (by omega) (by omega)]
+ · rw [ite_eq_right h2, ite_eq_right h2, hF _ (by omega) (by omega), hF _ h1.1 hx]
+ · rw [ite_eq_right h1, layF_get hblk F (by decide) zi (by omega) hx]
+ by_cases h3 : x < 16 * i
+ · rw [ite_eq_left (show x < 2 * 1 * (8 * (i + 1)) by omega), ite_eq_left (show x < 2 * 1 * (8 * i) by omega)]
+ · rw [ite_eq_right (show ¬ x < 2 * 1 * (8 * (i + 1)) by omega),
+ ite_eq_right (show ¬ x < 2 * 1 * (8 * i) by omega)]
+
+variable {bf : List Instr} (hbf : VBflyOk bf op)
+include hbf
+
+theorem ylay2_ok (hY : laneSseBlock (toY (gath2 ++ bf ++ scat2)) = some (gath2 ++ bf ++ scat2)) {fP sP : Addr}
+ (k : Nat) (o₀ o₁ : BitVec 8) (dz : BitVec 32) (zi kb : Nat → Nat) (hkb0 : kb 0 = k)
+ (hk : ∀ i < 16, kb i + 4 ≤ 256)
+ (hsel : ∀ i < 16, ∀ e < 4, kb i + sel o₀ e = zi (4 * i + 2 * (e / 2)) ∧
+ kb i + sel o₁ e = zi (4 * i + 1 + 2 * (e / 2)))
+ (hstep : ∀ i < 16, coeffAddr sP (kb i) + BitVec.signExtend 64 dz = coeffAddr sP (kb (i + 1)))
+ {F : Poly} {s : State} (hc : YConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP)
+ (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr)
+ (hd : (pR sP).Disjoint (pR fP)) :
+ WP isa (ylay2 bf k o₀ o₁ dz) s fun s' => PolyIs s'.mem fP (layF blk F 2 zi 64) ∧ BInvY fP s s' := by
+ have hk0 : k + 4 ≤ 256 := by have := hk 0 (by decide); omega
+ refine WP.seq (WP.mono (ypre21 k hk0 hdi hsi) fun w ⟨hdx, h8, og, hy⟩ => ?_)
+ have cw : YConsts w := ylanes_gpr (s := s) (og.lane hy) (YOnly.refl [] s) hc (by decide) (by decide)
+ refine WP.mono (wp_rcxLoopY (N := 16) (by decide) (by decide)
+ (fun i u => PolyIs u.mem fP (layF blk F 2 zi (4 * i)) ∧ u.gpr .rdx = coeffAddr fP (16 * i) ∧
+ u.gpr .r8 = coeffAddr sP (kb i) ∧ BInvY fP w u)
+ (fun u ou hu _ => ⟨by rw [ou.mem, og.mem]; exact hS,
+ by rw [ou.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero],
+ by rw [ou.keep.gpr (by decide), h8, hkb0], ⟨ou.keep.mono (by simp), by rw [ou.mem]; exact Frame.refl _ _,
+ ylanes_gpr (s := w) (ou.lane hu) (YOnly.refl [] w) cw (by decide) (by decide), ou.mxcsr⟩⟩)
+ (fun i hi u ⟨hS', hdx', h8', hb'⟩ => ?_)) fun u ⟨hS', _, _, hb'⟩ =>
+ ⟨hS', ⟨(og.keep.trans hb'.keep).mono (by simp), by rw [← og.mem]; exact hb'.frame, hb'.consts,
+ by rw [hb'.mxcsr, og.mxcsr]⟩⟩
+ have hT' : Tab zmTab u.mem sP 256 := (by rw [og.mem]; exact hT : Tab zmTab w.mem sP 256).frame hb'.frame
+ (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide)
+ have hwf' : pR fP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hwf
+ have hw' : pR sP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hw
+ have hk0' : ∀ j < 4, kb i + sel o₀ j < 256 := fun j _ => by have := sel_lt o₀ j; have := hk i hi; omega
+ have hk1' : ∀ j < 4, kb i + sel o₁ j < 256 := fun j _ => by have := sel_lt o₁ j; have := hk i hi; omega
+ rw [show [Instr.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm1 (at_ .rdx 32)] ++ yzetaS o₀ o₁ ++
+ [.alu .add .r8 (.imm dz)] ++ toY (gath2 ++ bf ++ scat2) ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1,
+ .alu .add .rdx (.imm 64)] ++ [.alu .sub .rcx (.imm 1)] =
+ ybody21 .xmm1 (yzetaS o₀ o₁) (gath2 ++ bf ++ scat2) dz by simp [List.append_assoc]]
+ have hR := fun x hx => layF2_next hblk F zi hi (x := x) hx
+ refine WP.mono (ystep21 hY (zs := [.xmm13, .xmm2, .xmm12]) (by decide) (by decide) (by decide) (by decide)
+ (by decide) (j := 16 * i) (by omega) (R := layF blk F 2 zi (4 * (i + 1)))
+ (ζ := fun l e => zetas (zi (4 * i + l + 2 * (e / 2)))) hb'.consts hdx' hS' hwf' (fun s' k' => ?_)
+ (fun l hl t ht hA hB hz ho => ?_) (fun x hx h => by rw [hR x hx, ite_eq_right (by omega)]))
+ fun u' ⟨hS'', hdx'', h8'', hcx, hzf, hb''⟩ => ⟨⟨hS'', by rw [hdx'', Nat.mul_succ],
+ by rw [h8'', h8', hstep i hi], hb'.trans hb''⟩, hcx, hzf⟩
+ · -- the zetas
+ refine WP.mono (yzetaS_ok o₀ o₁ (zP := sP) (k := kb i) hk0' hk1' (by rw [k'.gpr, h8'])
+ (by rw [k'.rd, k'.wr]; exact tab_in (List.mem_append_right _ hw') (hk i hi)) (by rw [k'.mem]; exact hT'))
+ fun s'' ⟨Z0, Z1, ZO, o⟩ => ⟨fun l hl => ⟨?_, ZO l hl⟩, o⟩
+ rcases lane01 hl with rfl | rfl
+ · exact Z0.congr fun e he => congrArg zetas ((hsel i hi e he).1.trans (congrArg zi (by omega)))
+ · exact Z1.congr fun e he => congrArg zetas ((hsel i hi e he).2.trans (congrArg zi (by omega)))
+ · -- the blocks of a lane
+ refine WP.mono (core2_ok hbf ht hA hB hz ho) fun t' ⟨⟨a, b⟩, o⟩ =>
+ ⟨⟨a.congr fun e he => ?_, b.congr fun e he => ?_⟩, o⟩
+ · by_cases h : e < 2
+ · rw [ite_eq_left h, hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)]
+ exact congrArg Prod.fst (op_idx op _ zi (by omega) (by omega) (by omega))
+ · rw [ite_eq_right h, hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)]
+ exact congrArg Prod.snd (op_idx op _ zi (by omega) (by omega) (by omega))
+ · by_cases h : e < 2
+ · rw [ite_eq_left h, hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)]
+ exact congrArg Prod.fst (op_idx op _ zi (by omega) (by omega) (by omega))
+ · rw [ite_eq_right h, hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)]
+ exact congrArg Prod.snd (op_idx op _ zi (by omega) (by omega) (by omega))
+
+theorem ylay1_ok (hY : laneSseBlock (toY (gath1 ++ bf ++ scat1)) = some (gath1 ++ bf ++ scat1)) {fP sP : Addr}
+ (k : Nat) (zl : List Instr) (dz : BitVec 32) (zi kb : Nat → Nat) (hkb0 : kb 0 = k)
+ (hk : ∀ i < 16, kb i + 8 ≤ 256)
+ (hzl : ∀ i < 16, ∀ s : State, s.gpr .r8 = coeffAddr sP (kb i) →
+ InRegions (s.rd ++ s.wr) (coeffAddr sP (kb i)) 32 → Tab zmTab s.mem sP 256 →
+ WP isa (.block zl) s fun s' => (∀ l < 2, ZLanes (s'.lane .xmm13 l)
+ (fun e => zetas (zi (8 * i + 2 * l + e + 2 * (e / 2)))) ∧ ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧
+ YOnly [.xmm13, .xmm12] s s')
+ (hstep : ∀ i < 16, coeffAddr sP (kb i) + BitVec.signExtend 64 dz = coeffAddr sP (kb (i + 1)))
+ {F : Poly} {s : State} (hc : YConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP)
+ (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr)
+ (hd : (pR sP).Disjoint (pR fP)) :
+ WP isa (ylay1 bf k zl dz) s fun s' => PolyIs s'.mem fP (layF blk F 1 zi 128) ∧ BInvY fP s s' := by
+ have hk0 : k + 4 ≤ 256 := by have := hk 0 (by decide); omega
+ refine WP.seq (WP.mono (ypre21 k hk0 hdi hsi) fun w ⟨hdx, h8, og, hy⟩ => ?_)
+ have cw : YConsts w := ylanes_gpr (s := s) (og.lane hy) (YOnly.refl [] s) hc (by decide) (by decide)
+ refine WP.mono (wp_rcxLoopY (N := 16) (by decide) (by decide)
+ (fun i u => PolyIs u.mem fP (layF blk F 1 zi (8 * i)) ∧ u.gpr .rdx = coeffAddr fP (16 * i) ∧
+ u.gpr .r8 = coeffAddr sP (kb i) ∧ BInvY fP w u)
+ (fun u ou hu _ => ⟨by rw [ou.mem, og.mem]; exact hS,
+ by rw [ou.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero],
+ by rw [ou.keep.gpr (by decide), h8, hkb0], ⟨ou.keep.mono (by simp), by rw [ou.mem]; exact Frame.refl _ _,
+ ylanes_gpr (s := w) (ou.lane hu) (YOnly.refl [] w) cw (by decide) (by decide), ou.mxcsr⟩⟩)
+ (fun i hi u ⟨hS', hdx', h8', hb'⟩ => ?_)) fun u ⟨hS', _, _, hb'⟩ =>
+ ⟨hS', ⟨(og.keep.trans hb'.keep).mono (by simp), by rw [← og.mem]; exact hb'.frame, hb'.consts,
+ by rw [hb'.mxcsr, og.mxcsr]⟩⟩
+ have hT' : Tab zmTab u.mem sP 256 := (by rw [og.mem]; exact hT : Tab zmTab w.mem sP 256).frame hb'.frame
+ (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide)
+ have hwf' : pR fP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hwf
+ have hw' : pR sP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hw
+ rw [show [Instr.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm2 (at_ .rdx 32)] ++ zl ++
+ [.alu .add .r8 (.imm dz)] ++ toY (gath1 ++ bf ++ scat1) ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1,
+ .alu .add .rdx (.imm 64)] ++ [.alu .sub .rcx (.imm 1)] =
+ ybody21 .xmm2 zl (gath1 ++ bf ++ scat1) dz by simp [List.append_assoc]]
+ have hR := fun x hx => layF1_next hblk F zi hi (x := x) hx
+ refine WP.mono (ystep21 hY (zs := [.xmm13, .xmm12]) (by decide) (by decide) (by decide) (by decide)
+ (by decide) (j := 16 * i) (by omega) (R := layF blk F 1 zi (8 * (i + 1)))
+ (ζ := fun l e => zetas (zi (8 * i + 2 * l + e + 2 * (e / 2)))) hb'.consts hdx' hS' hwf'
+ (fun s' k' => hzl i hi s' (by rw [k'.gpr, h8'])
+ (by rw [k'.rd, k'.wr]; exact f_in32 (List.mem_append_right _ hw') (hk i hi)) (by rw [k'.mem]; exact hT'))
+ (fun l hl t ht hA hB hz ho => ?_) (fun x hx h => by rw [hR x hx, ite_eq_right (by omega)]))
+ fun u' ⟨hS'', hdx'', h8'', hcx, hzf, hb''⟩ => ⟨⟨hS'', by rw [hdx'', Nat.mul_succ],
+ by rw [h8'', h8', hstep i hi], hb'.trans hb''⟩, hcx, hzf⟩
+ -- the blocks of a lane
+ refine WP.mono (core1_ok hbf ht hA hB hz ho) fun t' ⟨⟨a, b⟩, o⟩ =>
+ ⟨⟨a.congr fun e he => ?_, b.congr fun e he => ?_⟩, o⟩
+ · by_cases h : e % 2 = 0
+ · rw [ite_eq_left h, hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)]
+ exact congrArg Prod.fst (op_idx op _ zi (by omega) (by omega) (by omega))
+ · rw [ite_eq_right h, hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)]
+ exact congrArg Prod.snd (op_idx op _ zi (by omega) (by omega) (by omega))
+ · by_cases h : e % 2 = 0
+ · rw [ite_eq_left h, hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)]
+ exact congrArg Prod.fst (op_idx op _ zi (by omega) (by omega) (by omega))
+ · rw [ite_eq_right h, hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)]
+ exact congrArg Prod.snd (op_idx op _ zi (by omega) (by omega) (by omega))
+
+end
+
+end VG.Proof.MlDsa.X86_64.Arith
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMul.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMul.lean
new file mode 100644
index 000000000..32c854216
--- /dev/null
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMul.lean
@@ -0,0 +1,392 @@
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YBase
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul
+
+/-!
+# ML-DSA on x86-64: `vg_mldsa_multiply_ntt_avx2` and `vg_mldsa_multiply_add_ntt_avx2`
+
+Untrusted: everything here is checked by Lean. As `vg_mldsa_multiply_ntt`
+and `vg_mldsa_multiply_add_ntt` (`Mul.lean`) on eight coefficients at a
+time: each iteration of the loop loads eight coefficients of `f`, `g` and
+`h` and, in each lane, does what the SSE2 code's `mulCore` (`mulAddCore`)
+does (`ylanes`), on the coefficients `8i + 4l` to `8i + 4l + 3` of lane
+`l`, and stores the eight results (`YMul.step`); the loop stores the first
+248 (`YMul.loop_ok`), and the last eight, from the coefficients of `h` in
+`ymm6` (`YMul.last`), are stored after MXCSR is loaded back
+(`YMul.fn_ok`).
+-/
+
+namespace VG.Proof.MlDsa.X86_64.Arith
+
+open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith
+open VG.Proof.MlDsa.Arith
+open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes yld_ok yconst_ok WP.keep writesOnly gprPreserved_of ifp ifn
+ ptr_step GOnly wp_rcxLoopY add_ofNat_zero lane_setReg lane_setFlags sx32 State.setMem_ymm xmm_setXmm)
+open VG.Impl.MlKem.X86_64 (xb xmov toY yconst rcxLoop)
+open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced)
+
+theorem lane_mulCore : laneSseBlock (toY mulCore) = some mulCore := by decide +kernel
+theorem lane_mulAddCore : laneSseBlock (toY mulAddCore) = some mulAddCore := by decide +kernel
+
+/-- The constants, and `2⁶⁴ mod q` in both lanes of `ymm11`. -/
+theorem ymulPro_ok (s : State) :
+ WP isa (.block ymulPro) s fun s' => YConsts s' ∧ (∀ l < 2, s'.lane .xmm11 l = r2V) ∧
+ (∀ l < 2, s'.lane .xmm6 l = s.lane .xmm6 l) ∧ Keep [.rax] s s' ∧ s'.mem = s.mem := by
+ rw [ymulPro, WP.block_append_iff]
+ refine WP.mono (yconsts_ok s) fun s1 ⟨c1, k1, m1, _, o1⟩ =>
+ WP.mono (yconst_ok .xmm11 _ s1) fun s2 ⟨l2, k2, m2, _, o2⟩ =>
+ ⟨fun l hl => ⟨?_, ?_⟩, fun l hl => by rw [l2 l hl]; decide,
+ fun l hl => by rw [o2 _ (by decide) l hl, o1 _ (by decide) (by decide) l hl],
+ (k1.trans k2).mono (by simp), m2.trans m1⟩
+ · rw [State.proj_xmm, o2 _ (by decide) l hl]; exact (c1 l hl).q
+ · rw [State.proj_xmm, o2 _ (by decide) l hl]; exact (c1 l hl).qinv
+
+namespace YMul
+
+/-- After `i` iterations: the first `8i` coefficients of `h` are `R`'s, the
+others as they were in `s₀`. -/
+structure Inv (h f g : Addr) (s₀ : State) (R : Poly) (i : Nat) (s : State) : Prop where
+ rdi : s.gpr .rdi = h + BitVec.ofNat 64 (32 * i)
+ rsi : s.gpr .rsi = f + BitVec.ofNat 64 (32 * i)
+ rdx : s.gpr .rdx = g + BitVec.ofNat 64 (32 * i)
+ rd : s.rd = s₀.rd
+ wr : s.wr = s₀.wr
+ c : YConsts s
+ r2 : ∀ l < 2, s.lane .xmm11 l = r2V
+ x6 : ∀ l < 2, s.lane .xmm6 l = s₀.lane .xmm6 l
+ frame : Frame [pR h] s₀.mem s.mem
+ done : ∀ k < 8 * i, (coeffAt s.mem h k).toNat = (R[k]!).val
+ rest : ∀ k < 256, 8 * i ≤ k → coeffAt s.mem h k = coeffAt s₀.mem h k
+
+section
+variable {h f g : Addr} {s₀ : State} (hwh : pR h ∈ s₀.wr) (hrf : pR f ∈ s₀.rd ++ s₀.wr)
+ (hrg : pR g ∈ s₀.rd ++ s₀.wr) (hdf : (pR h).Disjoint (pR f)) (hdg : (pR h).Disjoint (pR g))
+ {F G : Poly} (hF : ∀ k < 256, (coeffAt s₀.mem f k).toNat = (F[k]!).val)
+ (hG : ∀ k < 256, (coeffAt s₀.mem g k).toNat = (G[k]!).val)
+ {core : List Instr} {Fv : BitVec 128 → BitVec 128 → BitVec 128 → BitVec 128}
+ (hcore : ∀ s : State, VConsts s → s.xmm .xmm11 = r2V → WP isa (.block core) s fun s' =>
+ s'.xmm .xmm3 = Fv (s.xmm .xmm3) (s.xmm .xmm13) (s.xmm .xmm5) ∧ XOnly [.xmm12, .xmm3, .xmm2, .xmm4] s s')
+ (hY : laneSseBlock (toY core) = some core)
+ {R : Poly} {H : Nat → BitVec 32} (hH : ∀ k < 248, coeffAt s₀.mem h k = H k)
+ (hlane : ∀ i < 64, ∀ x y z : BitVec 128, (∀ e < 4, (dword x e).toNat = (F[4 * i + e]!).val) →
+ (∀ e < 4, (dword y e).toNat = (G[4 * i + e]!).val) → (∀ e < 4, dword z e = H (4 * i + e)) →
+ ∀ e < 4, (dword (Fv x y z) e).toNat = (R[4 * i + e]!).val)
+include hwh hrf hrg hdf hdg hF hG hcore hY hH hlane
+
+theorem step {i : Nat} (hi : i < 31) {s : State} (hI : Inv h f g s₀ R i s) :
+ WP isa (.block (ymulLoads ++ toY core ++ ymulTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) s fun s' =>
+ Inv h f g s₀ R (i + 1) s' ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by
+ have j0 : 8 * i + 8 ≤ 256 := by omega
+ have e1 : s.gpr .rsi + BitVec.ofNat 64 0 = coeffAddr f (8 * i) := by
+ rw [add_ofNat_zero, hI.rsi]; congr 2; omega
+ have e2 : s.gpr .rdx + BitVec.ofNat 64 0 = coeffAddr g (8 * i) := by
+ rw [add_ofNat_zero, hI.rdx]; congr 2; omega
+ have e3 : s.gpr .rdi + BitVec.ofNat 64 0 = coeffAddr h (8 * i) := by
+ rw [add_ofNat_zero, hI.rdi]; congr 2; omega
+ have mf : ∀ k < 256, coeffAt s.mem f k = coeffAt s₀.mem f k := fun k hk =>
+ coeffAt_frame hI.frame (by simpa using hdf.symm) (by rw [n_eq]; exact hk)
+ have mg : ∀ k < 256, coeffAt s.mem g k = coeffAt s₀.mem g k := fun k hk =>
+ coeffAt_frame hI.frame (by simpa using hdg.symm) (by rw [n_eq]; exact hk)
+ rw [show ymulLoads ++ toY core ++ ymulTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr) =
+ [.vmovdquLoad .l256 .xmm3 (at_ .rsi 0)] ++ [.vmovdquLoad .l256 .xmm13 (at_ .rdx 0)] ++
+ [.vmovdquLoad .l256 .xmm5 (at_ .rdi 0)] ++ (toY core ++ (ymulTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr)))
+ by simp [ymulLoads, List.append_assoc],
+ WP.block_append_iff, WP.block_append_iff, WP.block_append_iff]
+ refine WP.mono (yld_ok (by rw [e1, hI.rd, hI.wr]; exact ⟨_, hrf, pR_contains32 f j0⟩)) fun s1 ⟨L1, o1⟩ => ?_
+ refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr, e2, hI.rd, hI.wr]; exact ⟨_, hrg, pR_contains32 g j0⟩))
+ fun s2 ⟨L2, o2⟩ => ?_
+ have o12 := o1.trans o2
+ refine WP.mono (yld_ok (by
+ rw [o12.rd, o12.wr, o12.gpr, e3, hI.rd, hI.wr]; exact f_in32 (List.mem_append_right _ hwh) j0))
+ fun s3 ⟨L3, o3⟩ => ?_
+ have o13 := o12.trans o3
+ rw [WP.block_append_iff]
+ refine WP.mono (ylanes hY (P := fun l t =>
+ t.xmm .xmm3 = Fv ((s3.proj l).xmm .xmm3) ((s3.proj l).xmm .xmm13) ((s3.proj l).xmm .xmm5))
+ fun l hl => hcore _ (yonly_yconsts o13 hI.c (by decide) (by decide) l hl)
+ (by rw [State.proj_xmm, o13.lane _ (by decide) l hl]; exact hI.r2 l hl))
+ fun s4 ⟨B4, o4⟩ => ?_
+ have o14 := o13.trans o4
+ have g4 : s4.gpr .rdi = coeffAddr h (8 * i) := by rw [o14.gpr, ← e3, add_ofNat_zero]
+ have w0 : InRegions s4.wr (s4.gpr .rdi) 32 := by rw [o14.wr, g4, hI.wr]; exact f_in32 hwh j0
+ -- the lanes of the result
+ have hl : ∀ l < 2, ∀ e < 4, (dword (s4.lane .xmm3 l) e).toNat = (R[8 * i + 4 * l + e]!).val := by
+ intro l hl e he
+ rw [← State.proj_xmm, B4 l hl, State.proj_xmm, State.proj_xmm, State.proj_xmm,
+ o3.lane _ (by decide) l hl, o2.lane _ (by decide) l hl, o3.lane _ (by decide) l hl, L1 l hl,
+ L2 l hl, L3 l hl, o1.gpr, o1.mem, o2.gpr, o2.mem, o1.gpr, o1.mem, e1, e2, e3,
+ show 8 * i + 4 * l + e = 4 * (2 * i + l) + e by omega]
+ refine hlane (2 * i + l) (by omega) _ _ _ (fun e he => ?_) (fun e he => ?_) (fun e he => ?_) e he
+ · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, mf _ (by omega), hF _ (by omega),
+ show 8 * i + 4 * l + e = 4 * (2 * i + l) + e by omega]
+ · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, mg _ (by omega), hG _ (by omega),
+ show 8 * i + 4 * l + e = 4 * (2 * i + l) + e by omega]
+ · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, hI.rest _ (by omega) (by omega),
+ hH _ (by omega), show 8 * i + 4 * l + e = 4 * (2 * i + l) + e by omega]
+ simp only [ymulTail]
+ vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd,
+ State.setMem_ymm, w0, sx32]
+ refine ⟨⟨?_, ?_, ?_, ?_, ?_, fun l hl => ⟨?_, ?_⟩, fun l hl => ?_, fun l hl => ?_, ?_, fun k hk => ?_,
+ fun k hk hk' => ?_⟩, ?_⟩
+ · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr]
+ rw [o14.gpr, hI.rdi]; exact ptr_step _ i 32
+ · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr]
+ rw [o14.gpr, hI.rsi]; exact ptr_step _ i 32
+ · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr]
+ rw [o14.gpr, hI.rdx]; exact ptr_step _ i 32
+ · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; rw [o14.rd, hI.rd]
+ · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; rw [o14.wr, hI.wr]
+ · rw [State.proj_xmm]; simp only [lane_setReg, lane_setFlags, State.setMem_lane]
+ exact (yonly_yconsts o14 hI.c (by decide) (by decide) l hl).q
+ · rw [State.proj_xmm]; simp only [lane_setReg, lane_setFlags, State.setMem_lane]
+ exact (yonly_yconsts o14 hI.c (by decide) (by decide) l hl).qinv
+ · simp only [lane_setReg, lane_setFlags, State.setMem_lane]; rw [o14.lane _ (by decide) l hl]; exact hI.r2 l hl
+ · simp only [lane_setReg, lane_setFlags, State.setMem_lane]; rw [o14.lane _ (by decide) l hl]; exact hI.x6 l hl
+ · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem]
+ rw [g4, o14.mem]; exact hI.frame.writeW (List.mem_singleton_self _) _ (pR_contains32 _ j0)
+ · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem]
+ rw [g4, o14.mem, coeffAt_write256 _ _ j0 _ (by omega)]
+ split
+ · rename_i hk'
+ rw [State.ymm, extract_ymm _ _ (by omega)]
+ split
+ · rename_i h4
+ have := hl 0 (by decide) (k - 8 * i) h4
+ rw [show 8 * i + 4 * 0 + (k - 8 * i) = k by omega] at this
+ exact this
+ · rename_i h4
+ have := hl 1 (by decide) (k - 8 * i - 4) (by omega)
+ rw [show 8 * i + 4 * 1 + (k - 8 * i - 4) = k by omega] at this
+ exact this
+ · exact hI.done k (by omega)
+ · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem]
+ rw [g4, o14.mem, coeffAt_write256 _ _ j0 _ hk, ifn (by omega)]
+ exact hI.rest k hk (by omega)
+ · exact ⟨by rw [o14.gpr], by rw [o14.gpr]⟩
+
+theorem loop_ok (hdi : s₀.gpr .rdi = h) (hsi : s₀.gpr .rsi = f) (hdx : s₀.gpr .rdx = g) (hc : YConsts s₀)
+ (h11 : ∀ l < 2, s₀.lane .xmm11 l = r2V) :
+ WP isa (rcxLoop 31 (ymulLoads ++ toY core ++ ymulTail)) s₀ (Inv h f g s₀ R 31) :=
+ wp_rcxLoopY (N := 31) (by decide) (by decide) _ (fun u o hy _ =>
+ have lu : ∀ r l, u.lane r l = s₀.lane r l := fun r l => by simp only [State.lane]; rw [o.xmm, hy]
+ ⟨by rw [o.keep.gpr (by decide), hdi, Nat.mul_zero, add_ofNat_zero],
+ by rw [o.keep.gpr (by decide), hsi, Nat.mul_zero, add_ofNat_zero],
+ by rw [o.keep.gpr (by decide), hdx, Nat.mul_zero, add_ofNat_zero], o.keep.2.1, o.keep.2.2,
+ fun l hl => ⟨by rw [State.proj_xmm, lu]; exact (hc l hl).q, by rw [State.proj_xmm, lu]; exact (hc l hl).qinv⟩,
+ fun l hl => by rw [lu]; exact h11 l hl, fun l _ => lu _ l,
+ by rw [o.mem]; exact Frame.refl _ _, fun k hk => absurd hk (by omega), fun k _ _ => by rw [o.mem]⟩)
+ fun i hi u hI => step hwh hrf hrg hdf hdg hF hG hcore hY hH hlane hi hI
+
+omit hwh hH in
+/-- The last eight coefficients, with those of `h` in `ymm6`. -/
+theorem last {s : State} (hI : Inv h f g s₀ R 31 s)
+ (hz : ∀ l < 2, ∀ e < 4, dword (s₀.lane .xmm6 l) e = H (248 + 4 * l + e)) :
+ WP isa (.block (ymulLast core)) s fun s' =>
+ (∀ l < 2, ∀ e < 4, (dword (s'.lane .xmm3 l) e).toNat = (R[248 + 4 * l + e]!).val) ∧ s'.gpr = s.gpr ∧
+ s'.mem = s.mem ∧ s'.rd = s.rd ∧ s'.wr = s.wr := by
+ have j0 : 8 * 31 + 8 ≤ 256 := by decide
+ have e1 : s.gpr .rsi + BitVec.ofNat 64 0 = coeffAddr f 248 := by rw [add_ofNat_zero, hI.rsi]
+ have e2 : s.gpr .rdx + BitVec.ofNat 64 0 = coeffAddr g 248 := by rw [add_ofNat_zero, hI.rdx]
+ have mf : ∀ k < 256, coeffAt s.mem f k = coeffAt s₀.mem f k := fun k hk =>
+ coeffAt_frame hI.frame (by simpa using hdf.symm) (by rw [n_eq]; exact hk)
+ have mg : ∀ k < 256, coeffAt s.mem g k = coeffAt s₀.mem g k := fun k hk =>
+ coeffAt_frame hI.frame (by simpa using hdg.symm) (by rw [n_eq]; exact hk)
+ have hY' : laneSseBlock ([.vop (.vmovdqa .l256 .xmm5 .xmm6)] ++ toY core) = some (xmov .xmm5 .xmm6 :: core) := by
+ show (match laneSse _, laneSseBlock (toY core) with | some a, some b => some (a ++ b) | _, _ => none) = _
+ rw [hY]; rfl
+ rw [ymulLast, show ∀ a b c : Instr, ∀ l : List Instr, [a, b, c] ++ l = [a] ++ [b] ++ ([c] ++ l) from
+ fun _ _ _ _ => rfl, WP.block_append_iff, WP.block_append_iff]
+ refine WP.mono (yld_ok (by rw [e1, hI.rd, hI.wr]; exact ⟨_, hrf, pR_contains32 f j0⟩)) fun s1 ⟨L1, o1⟩ => ?_
+ refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr, e2, hI.rd, hI.wr]; exact ⟨_, hrg, pR_contains32 g j0⟩))
+ fun s2 ⟨L2, o2⟩ => ?_
+ have o12 := o1.trans o2
+ refine WP.mono (ylanes hY' (rs := [.xmm5, .xmm12, .xmm3, .xmm2, .xmm4]) (P := fun l t =>
+ t.xmm .xmm3 = Fv ((s2.proj l).xmm .xmm3) ((s2.proj l).xmm .xmm13) ((s2.proj l).xmm .xmm6))
+ fun l hl => ?_) fun s3 ⟨B3, o3⟩ => ⟨fun l hl e he => ?_, (o12.trans o3).gpr, (o12.trans o3).mem,
+ (o12.trans o3).rd, (o12.trans o3).wr⟩
+ · have c2 := yonly_yconsts o12 hI.c (by decide) (by decide) l hl
+ rw [show xmov .xmm5 .xmm6 :: core = [xmov .xmm5 .xmm6] ++ core from rfl, WP.block_append_iff]
+ vrund [eval_movdqa]
+ refine WP.mono (hcore _ (c2.setXmm (by decide) (by decide) _)
+ (by rw [xmm_setXmm, ifn (by decide), State.proj_xmm, o12.lane _ (by decide) l hl]; exact hI.r2 l hl))
+ fun t ⟨ht, ot⟩ => ⟨?_, ?_⟩
+ · rw [ht, xmm_setXmm, xmm_setXmm, xmm_setXmm, ifn (by decide), ifn (by decide), ifp rfl]
+ · refine (XOnly.trans (⟨⟨rfl, rfl, rfl, rfl, rfl⟩, fun r hr => ?_⟩ :
+ XOnly [.xmm5] (s2.proj l) ((s2.proj l).setXmm .xmm5 ((s2.proj l).xmm .xmm6))) ot).mono
+ (rs' := [.xmm5, .xmm12, .xmm3, .xmm2, .xmm4]) (by simp)
+ rw [xmm_setXmm, ifn (by simpa using hr)]
+ · rw [← State.proj_xmm, B3 l hl, State.proj_xmm, State.proj_xmm, State.proj_xmm,
+ o2.lane _ (by decide) l hl, L1 l hl, L2 l hl, o1.gpr, o1.mem, e1, e2,
+ o12.lane _ (by decide) l hl, hI.x6 l hl, show 248 + 4 * l + e = 4 * (62 + l) + e by omega]
+ refine hlane (62 + l) (by omega) _ _ _ (fun e he => ?_) (fun e he => ?_) (fun e he => ?_) e he
+ · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, mf _ (by omega), hF _ (by omega),
+ show 248 + 4 * l + e = 4 * (62 + l) + e by omega]
+ · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, mg _ (by omega), hG _ (by omega),
+ show 248 + 4 * l + e = 4 * (62 + l) + e by omega]
+ · rw [hz l hl e he, show 248 + 4 * l + e = 4 * (62 + l) + e by omega]
+
+end
+
+
+/-- The whole function, from its precondition, with a `core` whose lanes are
+those of `t`. -/
+theorem fn_ok {t : Poly → Poly → Poly → Poly} {hPre : Mem → Addr → Prop} {σ : State} (hp : (mulK t hPre).pre σ)
+ {core : List Instr} {Fv : BitVec 128 → BitVec 128 → BitVec 128 → BitVec 128}
+ (hcore : ∀ s : State, VConsts s → s.xmm .xmm11 = r2V → WP isa (.block core) s fun s' =>
+ s'.xmm .xmm3 = Fv (s.xmm .xmm3) (s.xmm .xmm13) (s.xmm .xmm5) ∧ XOnly [.xmm12, .xmm3, .xmm2, .xmm4] s s')
+ (hY : laneSseBlock (toY core) = some core)
+ (hlane : ∀ i < 64, ∀ x y z : BitVec 128,
+ (∀ e < 4, (dword x e).toNat = ((polyAt σ.mem (σ.gpr .rsi))[4 * i + e]!).val) →
+ (∀ e < 4, (dword y e).toNat = ((polyAt σ.mem (σ.gpr .rdx))[4 * i + e]!).val) →
+ (∀ e < 4, dword z e = coeffAt σ.mem (σ.gpr .rdi) (4 * i + e)) →
+ ∀ e < 4, (dword (Fv x y z) e).toNat = ((t (polyAt σ.mem (σ.gpr .rdi)) (polyAt σ.mem (σ.gpr .rsi))
+ (polyAt σ.mem (σ.gpr .rdx)))[4 * i + e]!).val)
+ (hk : writesOnly [.rax, .rdi, .rsi, .rdx, .rcx]
+ (.seq (.block ymulPro) (.seq (rcxLoop 31 (ymulLoads ++ toY core ++ ymulTail)) (.block (ymulLast core)))) =
+ true) :
+ WP isa (ymulFn core) σ fun s' => Keep [.r8, .rax, .r11, .rax, .rdi, .rsi, .rdx, .rcx] σ s' ∧
+ Frame [pR (σ.gpr .rdi)] σ.mem s'.mem ∧ (mulK t hPre).post σ s' := by
+ obtain ⟨hrd, hwr, hdf, hdg, -, -, -, -, redf, redg⟩ := hp
+ generalize eh : σ.gpr .rdi = h at *
+ generalize ef : σ.gpr .rsi = f at *
+ generalize eg : σ.gpr .rdx = g at *
+ let R := t (polyAt σ.mem h) (polyAt σ.mem f) (polyAt σ.mem g)
+ have hwh : pR h ∈ σ.wr := by rw [hwr]; exact List.mem_singleton_self _
+ simp only [ymulFn]
+ rw [show ∀ a b : Instr, [a, b] = [a] ++ [b] from fun _ _ => rfl]
+ refine WP.seq (WP.mono (Q := fun (s1 : State) => s1.gpr .r8 = h ∧
+ (∀ l < 2, s1.lane .xmm6 l = σ.mem.readW (coeffAddr h 248 + BitVec.ofNat 64 (16 * l)) 128) ∧
+ Keep [.r8] σ s1 ∧ s1.mem = σ.mem) ?_ fun s1 ⟨h8, h6, k1, m1⟩ => ?_)
+ · rw [WP.block_append_iff]
+ refine WP.mono (Q := fun (s0 : State) => s0.gpr .r8 = h ∧ s0.gpr .rdi = h ∧ GOnly [.r8] σ s0 ∧
+ s0.ymmHi = σ.ymmHi) (by vrund [eh, RegUpd.ymmHi_setReg]; gonlyd)
+ fun s0 ⟨h80, hd0, o0, y0⟩ => ?_
+ refine WP.mono (yld_ok (by
+ rw [hd0, o0.keep.2.1, o0.keep.2.2]
+ exact ⟨_, List.mem_append_right _ hwh, Offset.contains_base h (by omega) (by omega)⟩))
+ fun s1 ⟨L1, o1⟩ => ⟨by rw [o1.gpr, h80], fun l hl => by rw [L1 l hl, hd0, o0.mem],
+ ⟨fun r hr => by rw [o1.gpr, o0.keep.gpr hr], by rw [o1.rd, o0.keep.2.1], by rw [o1.wr, o0.keep.2.2]⟩,
+ by rw [o1.mem, o0.mem]⟩
+ have hw1 : pR h ∈ s1.wr := by rw [k1.2.2]; exact hwh
+ refine WP.seq (WP.mono (withMxcsrH_ok (r := .r8) ⟨by decide, by decide⟩ [.rax, .rdi, .rsi, .rdx, .rcx]
+ ⟨by decide, by decide⟩ h8 hw1 hk (Q := fun (s3 : State) => Keep [.rax, .r11, .rax, .rdi, .rsi, .rdx, .rcx] s1 s3 ∧
+ s3.gpr .rdi = coeffAddr h 248 ∧ Frame [pR h] σ.mem s3.mem ∧
+ (∀ k < 248, (coeffAt s3.mem h k).toNat = (R[k]!).val) ∧
+ ∀ l < 2, ∀ e < 4, (dword (s3.lane .xmm3 l) e).toNat = (R[248 + 4 * l + e]!).val)
+ fun s2 k2 f2 x2 y2 => ?_) fun s4 ⟨s3, ⟨kk, hdi, fr, dn, ln⟩, f4, k4, x4, y4⟩ => ?_)
+ · have fσ2 : Frame [mxH h] σ.mem s2.mem := by rw [← m1]; exact f2
+ have l2 : ∀ r l, s2.lane r l = s1.lane r l := fun r l => by simp only [State.lane]; rw [x2, y2]
+ refine WP.mono (WP.keep [.rax, .rdi, .rsi, .rdx, .rcx] (WP.seq (WP.mono (ymulPro_ok s2)
+ fun w ⟨cw, xw, x6, kw, mw⟩ => ?_) (Q := fun (s3 : State) => s3.gpr .rdi = coeffAddr h 248 ∧
+ Frame [pR h] σ.mem s3.mem ∧ (∀ k < 248, (coeffAt s3.mem h k).toNat = (R[k]!).val) ∧
+ ∀ l < 2, ∀ e < 4, (dword (s3.lane .xmm3 l) e).toNat = (R[248 + 4 * l + e]!).val)) hk)
+ fun s3 ⟨q3, kk⟩ => ⟨k2.trans kk, q3⟩
+ have gw : ∀ r, r ≠ .rax → r ≠ .r11 → r ≠ .r8 → w.gpr r = σ.gpr r := fun r h1 h2 h3 => by
+ rw [kw.gpr (by simpa using h1), k2.gpr (by simp [h1, h2]), k1.gpr (by simpa using h3)]
+ have rw' : w.rd = σ.rd ∧ w.wr = σ.wr :=
+ ⟨kw.2.1.trans (k2.2.1.trans k1.2.1), kw.2.2.trans (k2.2.2.trans k1.2.2)⟩
+ have fσw : Frame [mxH h] σ.mem w.mem := by rw [mw]; exact fσ2
+ have fσw' : Frame [pR h] σ.mem w.mem :=
+ Frame.sub fσw fun r hr => ⟨pR h, List.mem_singleton_self _, by
+ simp only [List.mem_singleton] at hr; subst hr; exact mxH_sub h⟩
+ refine WP.seq (WP.mono (loop_ok (s₀ := w) (h := h) (f := f) (g := g) (R := R) (F := polyAt σ.mem f)
+ (G := polyAt σ.mem g) (H := coeffAt σ.mem h)
+ (by rw [rw'.2]; exact hwh) (by rw [rw'.1, hrd]; simp) (by rw [rw'.1, hrd]; simp) hdf hdg
+ (fun k hk => by
+ rw [coeffAt_frame fσw' (by simpa using hdf.symm) (by rw [n_eq]; exact hk),
+ polyAt_val redf (by rw [n_eq]; exact hk)])
+ (fun k hk => by
+ rw [coeffAt_frame fσw' (by simpa using hdg.symm) (by rw [n_eq]; exact hk),
+ polyAt_val redg (by rw [n_eq]; exact hk)])
+ hcore hY (fun k hk => Mul.coeffAt_mxH fσw (by omega)) hlane
+ (by rw [gw _ (by decide) (by decide) (by decide), eh]) (by rw [gw _ (by decide) (by decide) (by decide), ef])
+ (by rw [gw _ (by decide) (by decide) (by decide), eg]) cw xw) fun s hI => ?_)
+ refine WP.mono (last (s₀ := w) (h := h) (f := f) (g := g) (R := R) (F := polyAt σ.mem f)
+ (G := polyAt σ.mem g) (H := coeffAt σ.mem h)
+ (by rw [rw'.1, hrd]; simp) (by rw [rw'.1, hrd]; simp) hdf hdg
+ (fun k hk => by
+ rw [coeffAt_frame fσw' (by simpa using hdf.symm) (by rw [n_eq]; exact hk),
+ polyAt_val redf (by rw [n_eq]; exact hk)])
+ (fun k hk => by
+ rw [coeffAt_frame fσw' (by simpa using hdg.symm) (by rw [n_eq]; exact hk),
+ polyAt_val redg (by rw [n_eq]; exact hk)])
+ hcore hY hlane hI (fun l hl e he => by
+ rw [x6 l hl, l2, h6 l hl, dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq]))
+ fun s3 ⟨ln, g3, m3, _, _⟩ => ?_
+ refine ⟨by rw [g3, hI.rdi], ?_, fun k hk => by rw [m3]; exact hI.done k (by omega), ln⟩
+ rw [m3]
+ exact Frame.trans fσw' hI.frame
+ · have k14 := (k1.trans kk).trans k4
+ have hdi4 : s4.gpr .rdi = coeffAddr h 248 := by rw [k4.gpr (by simp), hdi]
+ have wh4 : InRegions s4.wr (s4.gpr .rdi) 32 := by
+ rw [hdi4, k14.2.2]; exact f_in32 hwh (by omega)
+ have l4 : ∀ r l, s4.lane r l = s3.lane r l := fun r l => by simp only [State.lane]; rw [x4, y4]
+ simp only [yepi, List.singleton_append]
+ vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd,
+ State.setMem_ymm, wh4]
+ have f4' : Frame [pR h] s3.mem s4.mem := Frame.sub f4 fun r hr => ⟨pR h, List.mem_singleton_self _, by
+ simp only [List.mem_singleton] at hr; subst hr; exact mxH_sub h⟩
+ refine ⟨k14.mono (by simp), (fr.trans f4').writeW (List.mem_singleton_self _) _
+ (by rw [hdi4]; exact pR_contains32 h (by omega)), ?_⟩
+ dsimp only [mulK]
+ rw [eh, ef, eg, show ∀ s : State, (VOp.vzeroupper.exec s).mem = s.mem from fun _ => rfl, State.setMem_mem]
+ refine polyIs_of_toNat fun k hk => ?_
+ rw [n_eq] at hk
+ rw [hdi4, coeffAt_write256 _ _ (j := 248) (by decide) _ hk]
+ split
+ · rw [State.ymm, extract_ymm _ _ (by omega)]
+ split
+ · rename_i h1 h4
+ have := ln 0 (by decide) (k - 248) h4
+ rw [show 248 + 4 * 0 + (k - 248) = k by omega, ← l4] at this
+ exact this
+ · rename_i h1 h4
+ have := ln 1 (by decide) (k - 248 - 4) (by omega)
+ rw [show 248 + 4 * 1 + (k - 248 - 4) = k by omega, ← l4] at this
+ exact this
+ · rw [Mul.coeffAt_mxH f4 (by omega)]
+ exact dn k (by omega)
+
+end YMul
+
+/-! ## The functions -/
+
+theorem mulY_correct (s : State) (hs : mulK'.pre s) :
+ ∃ t s', Exec isa mulAvx2 s t s' ∧ abiPreserved s s' ∧ mulK'.post s s' := by
+ obtain ⟨t, s', he, hk, hf, hq⟩ := YMul.fn_ok hs (core := mulCore) (Fv := fun x y _ => mulV x y)
+ (fun s hc h11 => mulCore_ok hc h11) lane_mulCore
+ (fun i hi x y z hx hy _ e he => by
+ rw [mul_get _ _ (by rw [n_eq]; omega)]
+ exact mul_lane hx hy he)
+ (by decide +kernel)
+ exact ⟨t, s', he, abiPreserved_of_ctl (by decide +kernel) he (gprPreserved_of hk (by decide) hf
+ (by simpa using hs.2.2.2.2.1)), hq⟩
+
+theorem mulAddY_correct (s : State) (hs : mulAddK.pre s) :
+ ∃ t s', Exec isa mulAddAvx2 s t s' ∧ abiPreserved s s' ∧ mulAddK.post s s' := by
+ obtain ⟨t, s', he, hk, hf, hq⟩ := YMul.fn_ok hs (core := mulAddCore) (Fv := mulAddV)
+ (fun s hc h11 => mulAddCore_ok hc h11) lane_mulAddCore
+ (fun i hi x y z hx hy hz e he => by
+ rw [add_get _ _ (by rw [n_eq]; omega), mul_get _ _ (by rw [n_eq]; omega)]
+ exact mulAdd_lane hx hy (c := fun e => (polyAt s.mem (s.gpr .rdi))[4 * i + e]!)
+ (fun e he => by rw [hz e he, polyAt_val hs.2.2.2.2.2.2.2.1 (by rw [n_eq]; omega)]) he)
+ (by decide +kernel)
+ exact ⟨t, s', he, abiPreserved_of_ctl (by decide +kernel) he (gprPreserved_of hk (by decide) hf
+ (by simpa using hs.2.2.2.2.1)), hq⟩
+
+theorem mulY_ct : ConstantTime isa mulK'.pre mulK'.pub mulAvx2 :=
+ VG.Taint.constantTime (A := taint) mulτ mul_agree (by taint_decide)
+
+theorem mulAddY_ct : ConstantTime isa mulAddK.pre mulAddK.pub mulAddAvx2 :=
+ VG.Taint.constantTime (A := taint) mulτ mul_agree (by taint_decide)
+
+theorem mulY_verified : Verified X86_64.target mulAvx2 (Spec.MlDsa.mulContract X86_64.abi) :=
+ Verified.of_correct mulY_correct mulY_ct (by
+ mldsa_implies [Spec.MlDsa.mulContract, Spec.MlDsa.mulSig, mulK, X86_64.abi, X86_64.argRegs]
+ [mulSat] using mulSat)
+
+theorem mulAddY_verified : Verified X86_64.target mulAddAvx2 (Spec.MlDsa.mulAddContract X86_64.abi) :=
+ Verified.of_correct mulAddY_correct mulAddY_ct (by
+ mldsa_implies [Spec.MlDsa.mulAddContract, Spec.MlDsa.mulSig, mulK, X86_64.abi, X86_64.argRegs]
+ [mulSat] using mulSat)
+
+end VG.Proof.MlDsa.X86_64.Arith
diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YNtt.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YNtt.lean
new file mode 100644
index 000000000..4e06aaf10
--- /dev/null
+++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YNtt.lean
@@ -0,0 +1,419 @@
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YLay21
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv
+
+/-!
+# ML-DSA on x86-64: `vg_mldsa_ntt_avx2` and `vg_mldsa_inv_ntt_avx2`
+
+Untrusted: everything here is checked by Lean. As `vg_mldsa_ntt` and
+`vg_mldsa_inv_ntt` (`Ntt.lean`, `NttInv.lean`): ML-KEM's `withMxcsr` runs
+the code from any MXCSR and keeps what it does (`ymx_correct`); the
+prologue leaves the table of zetas in `scratch` and the constants in both
+lanes (`ypro_ok`); each layer is `nttLayer` or `nttInvLayer` (`ylay_ok`,
+`ylay4_ok`, `ylay2_ok`, `ylay1_ok`), `NTT⁻¹` then multiplies every
+coefficient by `8347681 = 256⁻¹ mod q` (`yscale_ok`), and `vzeroupper`
+keeps the memory (`LIY.epi`).
+-/
+
+namespace VG.Proof.MlDsa.X86_64.Arith
+
+open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith
+open VG.Proof.MlDsa.Arith
+open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes yld_ok yconst_ok ifp ifn sel GOnly WP.keep writesOnly
+ gprPreserved_of withMxcsr_ok mxR add_ofNat_zero wp_rcxLoopY lane_setReg lane_setFlags State.setMem_ymm sx32)
+open VG.Impl.MlKem.X86_64 (xb xmov toY rcxLoop yconst)
+open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt PolyIs zetas ntt nttInv)
+
+theorem lane_vbfly : laneSseBlock (toY vbfly) = some vbfly := by decide +kernel
+theorem lane_vibfly : laneSseBlock (toY vibfly) = some vibfly := by decide +kernel
+theorem lane_core2f : laneSseBlock (toY (gath2 ++ vbfly ++ scat2)) = some (gath2 ++ vbfly ++ scat2) := by
+ decide +kernel
+theorem lane_core2i : laneSseBlock (toY (gath2 ++ vibfly ++ scat2)) = some (gath2 ++ vibfly ++ scat2) := by
+ decide +kernel
+theorem lane_core1f : laneSseBlock (toY (gath1 ++ vbfly ++ scat1)) = some (gath1 ++ vbfly ++ scat1) := by
+ decide +kernel
+theorem lane_core1i : laneSseBlock (toY (gath1 ++ vibfly ++ scat1)) = some (gath1 ++ vibfly ++ scat1) := by
+ decide +kernel
+theorem lane_vmul3 : laneSseBlock (toY (vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ vcsub .xmm3 .xmm2)) =
+ some (vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ vcsub .xmm3 .xmm2) := by decide +kernel
+
+/-! ## The prologue and the layers -/
+
+/-- The table of zetas and the constants. -/
+theorem ypro_ok {sP : Addr} {s : State} (hsi : s.gpr .rsi = sP) (hw : pR sP ∈ s.wr) :
+ WP isa (.block ypro) s fun s' => Tab zmTab s'.mem sP 256 ∧ YConsts s' ∧
+ Frame [pR sP] s.mem s'.mem ∧ Keep [.r9, .rax] s s' ∧ s'.mxcsr = s.mxcsr := by
+ rw [ypro, WP.block_append_iff]
+ refine WP.mono (dwordTab_ok zmTab (fun k => Nat.lt_trans (zmTab_lt k) (by decide)) (by decide) hsi hw)
+ fun s1 ⟨hT, hf, k1, x1, _⟩ => WP.mono (yconsts_ok s1) fun s2 ⟨hc, k2, m2, x2, _⟩ =>
+ ⟨by rw [m2]; exact hT, hc, by rw [m2]; exact hf, (k1.trans k2).mono (by simp), by rw [x2, x1]⟩
+
+/-- Between the layers: the polynomial `F` at `fP`, the table at `sP`, and
+the constants in both lanes. -/
+structure LIY (fP sP : Addr) (s₀ : State) (F : Poly) (s : State) : Prop where
+ P : PolyIs s.mem fP F
+ T : Tab zmTab s.mem sP 256
+ c : YConsts s
+ keep : Keep [.rax, .rcx, .rdx, .r8] s₀ s
+ frame : Frame [pR fP] s₀.mem s.mem
+
+/-- A layer, then `c`. -/
+theorem LIY.seq {fP sP : Addr} {s₀ : State} (hdi : s₀.gpr .rdi = fP) (hsi : s₀.gpr .rsi = sP)
+ (hwf : pR fP ∈ s₀.wr) (hw : pR sP ∈ s₀.wr) (hd : (pR sP).Disjoint (pR fP)) {l c : Prog isa}
+ {F F' : Poly} {Q : State → Prop}
+ (hl : ∀ s, YConsts s → s.gpr .rdi = fP → s.gpr .rsi = sP → PolyIs s.mem fP F → Tab zmTab s.mem sP 256 →
+ pR fP ∈ s.wr → pR sP ∈ s.wr → WP isa l s fun s' => PolyIs s'.mem fP F' ∧ BInvY fP s s')
+ (hc : ∀ s, LIY fP sP s₀ F' s → WP isa c s Q) {s : State} (hI : LIY fP sP s₀ F s) :
+ WP isa (.seq l c) s Q :=
+ WP.seq (WP.mono (hl s hI.c (by rw [hI.keep.gpr (by decide), hdi]) (by rw [hI.keep.gpr (by decide), hsi]) hI.P
+ hI.T (by rw [hI.keep.2.2]; exact hwf) (by rw [hI.keep.2.2]; exact hw))
+ fun s' ⟨hS, hb⟩ => hc s' ⟨hS, hI.T.frame hb.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd)
+ (by decide), hb.consts, (hI.keep.trans hb.keep).mono (by decide), hI.frame.trans hb.frame⟩)
+
+/-- `vzeroupper` keeps the memory. -/
+theorem LIY.epi {fP sP : Addr} {s₀ : State} {F : Poly} {s : State} (hI : LIY fP sP s₀ F s) :
+ WP isa (.block yepi) s fun s' => PolyIs s'.mem fP F ∧ Frame [pR fP] s₀.mem s'.mem :=
+ WP.mono (Q := fun (u : State) => u.mem = s.mem) (by simp only [yepi]; vrund; rfl)
+ fun u hm => by rw [hm]; exact ⟨hI.P, hI.frame⟩
+
+/-- A layer of `NTT` with `len ≥ 8`, whose first zeta is `zetas k`. -/
+theorem yfwdLay_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) (len k : Nat)
+ (hlen : len ∈ [8, 16, 32, 64, 128]) (hk : 128 / len = k) {F : Poly} (s : State) (hc : YConsts s)
+ (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256)
+ (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (ylay vbfly len k 4) s fun s' => PolyIs s'.mem fP (nttLayer F len) ∧ BInvY fP s s' := by
+ rw [nttLayer_eq]
+ exact ylay_ok vbfly_spec lane_vbfly nttBlk_ok hlen 4 (fun c => 128 / len + c) (by rw [hk]; rfl)
+ (fun c hc => by
+ simp only [List.mem_cons, List.not_mem_nil, or_false] at hlen
+ rcases hlen with rfl | rfl | rfl | rfl | rfl <;> omega)
+ (fun c _ => step_fwd _ _ 1 (by decide)) hc hdi hsi hS hT hwf hw hd
+
+theorem yfwdLay4_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s)
+ (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256)
+ (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (ylay4 vbfly 32 0x00 0x55 8) s fun s' => PolyIs s'.mem fP (nttLayer F 4) ∧ BInvY fP s s' := by
+ have h0 : ∀ e < 4, sel 0x00 e = 0 := by decide
+ have h5 : ∀ e < 4, sel 0x55 e = 1 := by decide
+ rw [nttLayer_eq, show 128 / 4 = 32 from rfl]
+ exact ylay4_ok vbfly_spec lane_vbfly nttBlk_ok 32 0x00 0x55 8 (fun c => 32 + c) (fun m => 32 + 2 * m) rfl
+ (fun m _ => by omega) (fun m _ e he => ⟨by rw [h0 e he]; omega, by rw [h5 e he]; omega⟩)
+ (fun m _ => (step_fwd _ _ 2 (by decide)).trans (congrArg _ (by omega))) hc hdi hsi hS hT hwf hw hd
+
+theorem yfwdLay2_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s)
+ (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256)
+ (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (ylay2 vbfly 64 0xA0 0xF5 16) s fun s' => PolyIs s'.mem fP (nttLayer F 2) ∧ BInvY fP s s' := by
+ have hA : ∀ e < 4, sel 0xA0 e = 2 * (e / 2) := by decide
+ have hF : ∀ e < 4, sel 0xF5 e = 1 + 2 * (e / 2) := by decide
+ rw [nttLayer_eq, show 128 / 2 = 64 from rfl]
+ exact ylay2_ok nttBlk_ok vbfly_spec lane_core2f 64 0xA0 0xF5 16 (fun c => 64 + c) (fun i => 64 + 4 * i) rfl
+ (fun i _ => by omega) (fun i _ e he => ⟨by rw [hA e he]; omega, by rw [hF e he]; omega⟩)
+ (fun i _ => (step_fwd _ _ 4 (by decide)).trans (congrArg _ (by omega))) hc hdi hsi hS hT hwf hw hd
+
+theorem yfwdLay1_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s)
+ (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256)
+ (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (ylay1 vbfly 128 yzeta8 32) s fun s' => PolyIs s'.mem fP (nttLayer F 1) ∧ BInvY fP s s' := by
+ rw [nttLayer_eq, show 128 / 1 = 128 from rfl]
+ exact ylay1_ok nttBlk_ok vbfly_spec lane_core1f 128 yzeta8 32 (fun c => 128 + c) (fun i => 128 + 8 * i) rfl
+ (fun i _ => by omega)
+ (fun i hi s h8 hin hT => WP.mono (yzeta8_ok (by omega) h8 hin hT) fun _ ⟨z, o⟩ =>
+ ⟨fun l hl => ⟨(z l hl).1.congr fun e he => congrArg zetas (by omega), (z l hl).2⟩, o⟩)
+ (fun i _ => (step_fwd _ _ 8 (by decide)).trans (congrArg _ (by omega))) hc hdi hsi hS hT hwf hw hd
+
+theorem yinvLay_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) (len k : Nat)
+ (hlen : len ∈ [8, 16, 32, 64, 128]) (hk : 256 / len - 1 = k) {F : Poly} (s : State) (hc : YConsts s)
+ (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256)
+ (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (ylay vibfly len k (-4)) s fun s' => PolyIs s'.mem fP (nttInvLayer F len) ∧ BInvY fP s s' := by
+ have hl : 128 / len ≥ 1 ∧ 256 / len = 2 * (128 / len) ∧ 256 / len ≤ 32 := by
+ simp only [List.mem_cons, List.not_mem_nil, or_false] at hlen
+ rcases hlen with rfl | rfl | rfl | rfl | rfl <;> decide
+ rw [nttInvLayer_eq]
+ exact ylay_ok vibfly_spec lane_vibfly nttInvBlk_ok hlen (-4) (fun c => 256 / len - 1 - c) (by rw [hk]; rfl)
+ (fun c _ => by omega)
+ (fun c hc' => (congrArg (· + _) (congrArg (coeffAddr sP) (show 256 / len - 1 - c =
+ 256 / len - 1 - (c + 1) + 1 by omega))).trans (step_bwd _ _ 1 (by decide)))
+ hc hdi hsi hS hT hwf hw hd
+
+theorem yinvLay4_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s)
+ (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256)
+ (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (ylay4 vibfly 62 0x55 0x00 (-8)) s fun s' => PolyIs s'.mem fP (nttInvLayer F 4) ∧ BInvY fP s s' := by
+ have h0 : ∀ e < 4, sel 0x00 e = 0 := by decide
+ have h5 : ∀ e < 4, sel 0x55 e = 1 := by decide
+ rw [nttInvLayer_eq, show 256 / 4 - 1 = 63 from rfl, show 128 / 4 = 32 from rfl]
+ exact ylay4_ok vibfly_spec lane_vibfly nttInvBlk_ok 62 0x55 0x00 (-8) (fun c => 63 - c) (fun m => 62 - 2 * m)
+ rfl (fun m _ => by omega) (fun m _ e he => ⟨by rw [h5 e he]; omega, by rw [h0 e he]; omega⟩)
+ (fun m _ => (congrArg (· + _) (congrArg (coeffAddr sP) (show 62 - 2 * m = 62 - 2 * (m + 1) + 2 by
+ omega))).trans (step_bwd _ _ 2 (by decide))) hc hdi hsi hS hT hwf hw hd
+
+theorem yinvLay2_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s)
+ (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256)
+ (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (ylay2 vibfly 124 0x5F 0x0A (-16)) s fun s' => PolyIs s'.mem fP (nttInvLayer F 2) ∧
+ BInvY fP s s' := by
+ have hA : ∀ e < 4, sel 0x5F e = 3 - 2 * (e / 2) := by decide
+ have hB : ∀ e < 4, sel 0x0A e = 2 - 2 * (e / 2) := by decide
+ rw [nttInvLayer_eq, show 256 / 2 - 1 = 127 from rfl, show 128 / 2 = 64 from rfl]
+ exact ylay2_ok nttInvBlk_ok vibfly_spec lane_core2i 124 0x5F 0x0A (-16) (fun c => 127 - c)
+ (fun i => 124 - 4 * i) rfl (fun i _ => by omega)
+ (fun i _ e he => ⟨by rw [hA e he]; omega, by rw [hB e he]; omega⟩)
+ (fun i _ => (congrArg (· + _) (congrArg (coeffAddr sP) (show 124 - 4 * i = 124 - 4 * (i + 1) + 4 by
+ omega))).trans (step_bwd _ _ 4 (by decide))) hc hdi hsi hS hT hwf hw hd
+
+theorem yinvLay1_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s)
+ (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256)
+ (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) :
+ WP isa (ylay1 vibfly 248 yzeta8R (-32)) s fun s' => PolyIs s'.mem fP (nttInvLayer F 1) ∧
+ BInvY fP s s' := by
+ rw [nttInvLayer_eq, show 256 / 1 - 1 = 255 from rfl, show 128 / 1 = 128 from rfl]
+ exact ylay1_ok nttInvBlk_ok vibfly_spec lane_core1i 248 yzeta8R (-32) (fun c => 255 - c)
+ (fun i => 248 - 8 * i) rfl (fun i _ => by omega)
+ (fun i hi s h8 hin hT => WP.mono (yzeta8R_ok (by omega) h8 hin hT) fun _ ⟨z, o⟩ =>
+ ⟨fun l hl => ⟨(z l hl).1.congr fun e he => congrArg zetas (by omega), (z l hl).2⟩, o⟩)
+ (fun i _ => (congrArg (· + _) (congrArg (coeffAddr sP) (show 248 - 8 * i = 248 - 8 * (i + 1) + 8 by
+ omega))).trans (step_bwd _ _ 8 (by decide))) hc hdi hsi hS hT hwf hw hd
+
+/-! ## The multiplication by `256⁻¹` -/
+
+/-- The coefficients of `G` before `8i` multiplied by `8347681`. -/
+def YScaled (m : Mem) (fP : Addr) (G : Poly) (i : Nat) : Prop :=
+ ∀ k < 256, (coeffAt m fP k).toNat = (if k < 8 * i then G[k]! * 8347681 else G[k]!).val
+
+/-- `8347681 · 2³² mod q` in each doubleword. -/
+theorem scale_lanes : ZLanes (ofDwords 16382#32 16382#32 16382#32 16382#32) (fun _ => 8347681) ∧
+ ZOdd (ofDwords 16382#32 16382#32 16382#32 16382#32) (ofDwords 16382#32 16382#32 16382#32 16382#32) :=
+ ⟨fun i hi => by rcases cases4 hi with rfl | rfl | rfl | rfl <;> decide,
+ fun j hj => by rcases (by omega : j = 0 ∨ j = 1) with rfl | rfl <;> decide⟩
+
+theorem mov12_ok (t : State) :
+ WP isa (.block [xmov .xmm12 .xmm13]) t fun t' => t'.xmm .xmm12 = t.xmm .xmm13 ∧ XOnly [.xmm12] t t' := by
+ simp only [xmov, xb]
+ vrun [eval_movdqa]
+ exact ⟨by first | trivial | simp, by xonly⟩
+
+/-- The body of the loop of `yscale`. -/
+abbrev sbodyY : List Instr :=
+ [.vmovdquLoad .l256 .xmm3 (at_ .rdx 0)] ++ toY (vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ vcsub .xmm3 .xmm2) ++
+ [.vmovdquStore .l256 (at_ .rdx 0) .xmm3, .alu .add .rdx (.imm 32)] ++ [.alu .sub .rcx (.imm 1)]
+
+theorem yscale_step {fP : Addr} {G : Poly} {i : Nat} (hi : i < 32) {s : State} (hc : YConsts s)
+ (hz : ∀ l < 2, ZLanes (s.lane .xmm13 l) (fun _ => 8347681))
+ (ho : ∀ l < 2, ZOdd (s.lane .xmm13 l) (s.lane .xmm12 l))
+ (hdx : s.gpr .rdx = coeffAddr fP (8 * i)) (hS : YScaled s.mem fP G i) (hw : pR fP ∈ s.wr) :
+ WP isa (.block sbodyY) s fun s' =>
+ YScaled s'.mem fP G (i + 1) ∧ s'.gpr .rdx = coeffAddr fP (8 * (i + 1)) ∧
+ Frame [pR fP] s.mem s'.mem ∧ YConsts s' ∧ (∀ l < 2, s'.lane .xmm13 l = s.lane .xmm13 l) ∧
+ (∀ l < 2, s'.lane .xmm12 l = s.lane .xmm12 l) ∧ Keep [.rdx, .rcx] s s' ∧
+ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) ∧ s'.mxcsr = s.mxcsr := by
+ have j0 : 8 * i + 8 ≤ 256 := by omega
+ have r0 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 0) 32 := by
+ rw [hdx, add_ofNat_zero]; exact f_in32 (List.mem_append_right _ hw) j0
+ rw [sbodyY, List.append_assoc, List.append_assoc, WP.block_append_iff]
+ refine WP.mono (yld_ok r0) fun s1 ⟨L1, o1⟩ => ?_
+ rw [WP.block_append_iff]
+ refine WP.mono (ylanes lane_vmul3 (P := fun l t =>
+ t.xmm .xmm3 = csubV (montV (s1.lane .xmm3 l) (s1.lane .xmm13 l) (s1.lane .xmm12 l)))
+ fun l hl => vmul3_ok (yonly_yconsts o1 hc (by decide) (by decide) l hl)) fun s2 ⟨V2, o2⟩ => ?_
+ have o12 := o1.trans o2
+ have hv : ∀ l < 2, ∀ e < 4, (dword (s2.lane .xmm3 l) e).toNat = (G[8 * i + 4 * l + e]! * 8347681).val := by
+ intro l hl e he
+ have hx : DLanes (s1.lane .xmm3 l) (fun e => G[8 * i + 4 * l + e]!) := fun e he => by
+ rw [L1 l hl, hdx, add_ofNat_zero, dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq,
+ hS _ (by omega), ifn (by omega)]
+ have hz' : ZLanes (s1.lane .xmm13 l) (fun _ => 8347681) := by
+ rw [o1.lane _ (by decide) l hl]; exact hz l hl
+ have ho' : ZOdd (s1.lane .xmm13 l) (s1.lane .xmm12 l) := by
+ rw [o1.lane _ (by decide) l hl, o1.lane _ (by decide) l hl]; exact ho l hl
+ have e2 : s2.lane .xmm3 l = _ := V2 l hl
+ rw [e2, dword_csubV _ he, mulZ (hx e he) (hz' e he) (dword_montV ho' (prod_lt hx hz') he)]
+ dsimp only
+ rw [Fin.mul_comm]
+ have w0 : InRegions s2.wr (s2.gpr .rdx) 32 := by rw [o12.wr, o12.gpr, hdx]; exact f_in32 hw j0
+ vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, State.setMem_ymm,
+ w0, sx32]
+ have g2 : s2.gpr .rdx = coeffAddr fP (8 * i) := by rw [o12.gpr, hdx]
+ rw [g2, o12.mem]
+ refine ⟨fun k hk => ?_, ?_, ?_, ?_, ?_, ?_, ⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, ?_⟩
+ · rw [coeffAt_write256 _ _ j0 _ hk]
+ split
+ · rename_i h
+ rw [ifp (show k < 8 * (i + 1) by omega), State.ymm, extract_ymm _ _ (by omega)]
+ split
+ · have := hv 0 (by decide) (k - 8 * i) (by omega)
+ rw [show 8 * i + 4 * 0 + (k - 8 * i) = k by omega] at this; exact this
+ · have := hv 1 (by decide) (k - 8 * i - 4) (by omega)
+ rw [show 8 * i + 4 * 1 + (k - 8 * i - 4) = k by omega] at this; exact this
+ · rename_i h
+ rw [hS k hk]
+ by_cases h' : k < 8 * i
+ · rw [ifp h', ifp (by omega)]
+ · rw [ifn h', ifn (by omega)]
+ · rw [show (32 : BitVec 64) = BitVec.ofNat 64 (4 * 8) from rfl, coeffAddr_add,
+ show 8 * i + 8 = 8 * (i + 1) by omega]
+ · exact (Frame.refl _ _).writeW (List.mem_singleton_self _) _ (pR_contains32 fP j0)
+ · exact ylanes_gpr (s := s2) (fun r l => by simp only [lane_setReg, lane_setFlags, State.setMem_lane]) o12 hc
+ (by decide) (by decide)
+ · intro l hl; simp only [lane_setReg, lane_setFlags, State.setMem_lane]; exact o12.lane _ (by decide) l hl
+ · intro l hl; simp only [lane_setReg, lane_setFlags, State.setMem_lane]; exact o12.lane _ (by decide) l hl
+ · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr
+ simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr.1, hr.2, ite_false, State.setMem_gpr]
+ rw [o12.gpr]
+ · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; rw [o12.rd]
+ · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; rw [o12.wr]
+ · rw [o12.gpr]
+ · rw [o12.gpr]
+ · exact o12.mxcsr
+
+/-- Every coefficient times `8347681`. -/
+theorem yscale_ok {fP sP : Addr} (_hd : (pR sP).Disjoint (pR fP)) {G : Poly} (s : State) (hc : YConsts s)
+ (hdi : s.gpr .rdi = fP) (_hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP G) (_hT : Tab zmTab s.mem sP 256)
+ (hwf : pR fP ∈ s.wr) (_hw : pR sP ∈ s.wr) :
+ WP isa yscale s fun s' => PolyIs s'.mem fP (G.map (· * 8347681)) ∧ BInvY fP s s' := by
+ refine WP.seq ?_
+ rw [WP.block_append_iff, WP.block_append_iff]
+ refine WP.mono (Q := fun (w : State) => w.gpr .rdx = fP ∧ GOnly [.rdx] s w ∧ w.ymmHi = s.ymmHi)
+ (by vrund [hdi]; exact ⟨by gonlyd, rfl⟩) fun w ⟨hdx, og, hy⟩ => ?_
+ refine WP.mono (yconst_ok .xmm13 16382 w) fun w1 ⟨l1, k1, m1, x1, o1⟩ => ?_
+ refine WP.mono (ylanes (by decide) (P := fun l t => t.xmm .xmm12 = w1.lane .xmm13 l)
+ fun l _ => mov12_ok (w1.proj l)) fun w2 ⟨l2, o2⟩ => ?_
+ have cw : YConsts w := ylanes_gpr (s := s) (og.lane hy) (YOnly.refl [] s) hc (by decide) (by decide)
+ have cw2 : YConsts w2 := yonly_yconsts o2 (fun l hl =>
+ ⟨by rw [State.proj_xmm, o1 _ (by decide) l hl]; exact (cw l hl).q,
+ by rw [State.proj_xmm, o1 _ (by decide) l hl]; exact (cw l hl).qinv⟩) (by decide) (by decide)
+ have z13 : ∀ l < 2, w2.lane .xmm13 l = ofDwords 16382#32 16382#32 16382#32 16382#32 := fun l hl => by
+ rw [o2.lane _ (by decide) l hl, l1 l hl]; rfl
+ have z12 : ∀ l < 2, w2.lane .xmm12 l = ofDwords 16382#32 16382#32 16382#32 16382#32 := fun l hl => by
+ have e : w2.lane .xmm12 l = _ := l2 l hl
+ rw [e, l1 l hl]; rfl
+ have dx2 : w2.gpr .rdx = fP := by rw [o2.gpr, k1.gpr (by decide), hdx]
+ have mw2 : w2.mem = s.mem := by rw [o2.mem, m1, og.mem]
+ refine WP.mono (wp_rcxLoopY (N := 32) (by decide) (by decide)
+ (fun i u => YScaled u.mem fP G i ∧ u.gpr .rdx = coeffAddr fP (8 * i) ∧ YConsts u ∧
+ (∀ l < 2, u.lane .xmm13 l = w2.lane .xmm13 l) ∧ (∀ l < 2, u.lane .xmm12 l = w2.lane .xmm12 l) ∧
+ Keep [.rcx, .rdx] w2 u ∧ Frame [pR fP] w2.mem u.mem ∧ u.mxcsr = w2.mxcsr)
+ (fun u o hu _ => ⟨fun k hk => by
+ rw [o.mem, mw2, ifn (by omega)]; exact polyIs_toNat hS (by rw [n_eq]; exact hk),
+ by rw [o.keep.gpr (by decide), dx2, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero],
+ ylanes_gpr (s := w2) (o.lane hu) (YOnly.refl [] w2) cw2 (by decide) (by decide),
+ fun l _ => o.lane hu _ l, fun l _ => o.lane hu _ l, o.keep.mono (by simp),
+ by rw [o.mem]; exact Frame.refl _ _, o.mxcsr⟩)
+ (fun i hi u ⟨hS', hdx', hc', hz', hzo', hk', hf', hx'⟩ => WP.mono (yscale_step hi hc'
+ (fun l hl => by rw [hz' l hl, z13 l hl]; exact scale_lanes.1)
+ (fun l hl => by rw [hz' l hl, hzo' l hl, z13 l hl, z12 l hl]; exact scale_lanes.2) hdx' hS'
+ (by rw [hk'.2.2, o2.wr, k1.2.2, og.keep.2.2]; exact hwf))
+ fun u' ⟨hS'', hdx'', hf'', hc'', hz'', hzo'', hk'', hcx, hzf, hx''⟩ =>
+ ⟨⟨hS'', hdx'', hc'', fun l hl => by rw [hz'' l hl, hz' l hl], fun l hl => by rw [hzo'' l hl, hzo' l hl],
+ (hk'.trans hk'').mono (by simp), hf'.trans hf'', by rw [hx'', hx']⟩, hcx, hzf⟩))
+ fun u ⟨hS', _, hc', _, _, hk', hf', hx'⟩ => ?_
+ have kw2 : Keep [.rdx, .rax] s w2 :=
+ ((og.keep.trans k1).trans (⟨fun r _ => by rw [o2.gpr], o2.rd, o2.wr⟩ : Keep [] w1 w2)).mono (by simp)
+ refine ⟨polyIs_of_toNat fun k hk => ?_, ⟨(kw2.trans hk').mono (by simp), by rw [← mw2]; exact hf', hc',
+ by rw [hx', o2.mxcsr, x1, og.mxcsr]⟩⟩
+ rw [n_eq] at hk
+ rw [hS' k hk, ifp (by omega), map_mul_get _ _ (by rw [n_eq]; exact hk)]
+
+/-! ## The functions -/
+
+/-- The code in `withMxcsr`, from its state `s1`: the prologue, then the
+layers `l`, which leave `G`. -/
+theorem ynttBody_ok {t : Poly → Poly} {s s1 : State} (hs : (inPlaceK t).pre s) {l : Prog isa} {G : Poly}
+ (k1 : Keep [.rax, .r11] s s1) (f1 : Frame [mxR (s.gpr .rsi)] s.mem s1.mem)
+ (hl : ∀ s2, LIY (s.gpr .rdi) (s.gpr .rsi) s2 (polyAt s.mem (s.gpr .rdi)) s2 → s2.gpr .rdi = s.gpr .rdi →
+ s2.gpr .rsi = s.gpr .rsi → pR (s.gpr .rdi) ∈ s2.wr → pR (s.gpr .rsi) ∈ s2.wr →
+ WP isa l s2 fun s3 => PolyIs s3.mem (s.gpr .rdi) G ∧ Frame [pR (s.gpr .rdi)] s2.mem s3.mem) :
+ WP isa (.seq (.block ypro) l) s1 fun s' =>
+ PolyIs s'.mem (s.gpr .rdi) G ∧ Frame [pR (s.gpr .rdi), pR (s.gpr .rsi)] s.mem s'.mem := by
+ have hw : pR (s.gpr .rsi) ∈ s.wr := by rw [hs.2.1]; simp
+ have hwf : pR (s.gpr .rdi) ∈ s.wr := by rw [hs.2.1]; simp
+ have hd : (pR (s.gpr .rdi)).Disjoint (pR (s.gpr .rsi)) := hs.2.2.1
+ have hdi1 : s1.gpr .rdi = s.gpr .rdi := k1.gpr (by decide)
+ have hsi1 : s1.gpr .rsi = s.gpr .rsi := k1.gpr (by decide)
+ have hF1 : PolyIs s1.mem (s.gpr .rdi) (polyAt s.mem (s.gpr .rdi)) :=
+ polyIs_frame f1 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd.sub_right (mx_sub' _))
+ ⟨hs.2.2.2.2.2, rfl⟩
+ refine WP.seq (WP.mono (ypro_ok hsi1 (by rw [k1.2.2]; exact hw)) fun s2 ⟨hT, hc, hf2, k2, _⟩ => ?_)
+ have hF2 : PolyIs s2.mem (s.gpr .rdi) (polyAt s.mem (s.gpr .rdi)) :=
+ polyIs_frame hf2 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) hF1
+ refine WP.mono (hl s2 ⟨hF2, hT, hc, Keep.refl _ _, Frame.refl _ _⟩
+ (by rw [k2.gpr (by decide), hdi1]) (by rw [k2.gpr (by decide), hsi1])
+ (by rw [k2.2.2, k1.2.2]; exact hwf) (by rw [k2.2.2, k1.2.2]; exact hw)) fun s3 ⟨hP, hf3⟩ => ⟨hP, ?_⟩
+ refine (frame_fs f1 ?_).trans ((frame_fs hf2 ?_).trans (frame_fs hf3 ?_)) <;>
+ intro r hr <;> simp only [List.mem_singleton] at hr <;> subst hr
+ exacts [.inr (mx_sub' _), .inr fun _ h => h, .inl fun _ h => h]
+
+/-- `withMxcsr` around the body, and the ABI. -/
+theorem ymx_correct {t : Poly → Poly} {l : Prog isa} (s : State) (hs : (inPlaceK t).pre s)
+ (hk : writesOnly [.rax, .rcx, .rdx, .r8, .r9] (.seq (.block ypro) l) = true)
+ (hctl : ctlOk (VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 (.seq (.block ypro) l)) = true)
+ (hk' : writesOnly [.rax, .rcx, .rdx, .r8, .r9, .r11]
+ (VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 (.seq (.block ypro) l)) = true)
+ (hl : ∀ s1, Keep [.rax, .r11] s s1 → Frame [mxR (s.gpr .rsi)] s.mem s1.mem →
+ WP isa (.seq (.block ypro) l) s1 fun s' => PolyIs s'.mem (s.gpr .rdi) (t (polyAt s.mem (s.gpr .rdi))) ∧
+ Frame [pR (s.gpr .rdi), pR (s.gpr .rsi)] s.mem s'.mem) :
+ ∃ tr s', Exec isa (VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 (.seq (.block ypro) l)) s tr s' ∧
+ abiPreserved s s' ∧ (inPlaceK t).post s s' := by
+ have hw : pR (s.gpr .rsi) ∈ s.wr := by rw [hs.2.1]; simp
+ have hd : (pR (s.gpr .rdi)).Disjoint (pR (s.gpr .rsi)) := hs.2.2.1
+ have hW := withMxcsr_ok (c := .seq (.block ypro) l) (by decide) [.rax, .rcx, .rdx, .r8, .r9] (by decide) rfl hw
+ hk (hl)
+ obtain ⟨tr, s', he, ⟨s2, ⟨hP, hf⟩, hf', -⟩, hk⟩ := WP.keep [.rax, .rcx, .rdx, .r8, .r9, .r11] hW hk'
+ refine ⟨tr, s', he, abiPreserved_of_ctl hctl he (gprPreserved_of hk (by decide)
+ (hf.trans (hf'.sub fun r hr => ⟨_, List.mem_cons_of_mem _ (List.mem_singleton_self _), ?_⟩))
+ (by simpa using ⟨hs.2.2.2.1, hs.2.2.2.2.1⟩)), ?_⟩
+ · rw [List.mem_singleton.mp hr]; exact mx_sub' _
+ · exact polyIs_frame hf' (fun r hr => by
+ rw [List.mem_singleton.mp hr]; exact hd.sub_right (mx_sub' _)) hP
+
+theorem nttY_correct (s : State) (hs : (inPlaceK ntt).pre s) :
+ ∃ t s', Exec isa nttAvx2 s t s' ∧ abiPreserved s s' ∧ (inPlaceK ntt).post s s' := by
+ have hd : (pR (s.gpr .rsi)).Disjoint (pR (s.gpr .rdi)) := hs.2.2.1.symm
+ refine ymx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 =>
+ WP.mono (ynttBody_ok hs k1 f1 (G := nttLens.foldl nttLayer (polyAt s.mem (s.gpr .rdi)))
+ fun s2 hI hdi hsi hwf hw => ?_) fun s' ⟨hP, hf⟩ => ⟨by rw [ntt_eq_layers]; exact hP, hf⟩
+ simp only [nttLens, List.foldl_cons, List.foldl_nil]
+ refine LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 128 1 (by decide) (by decide)) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 64 2 (by decide) (by decide)) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 32 4 (by decide) (by decide)) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 16 8 (by decide) (by decide)) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 8 16 (by decide) (by decide)) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay4_ok hd) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay2_ok hd) ?_ hI
+ exact fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay1_ok hd) (fun _ hI => hI.epi) hI
+
+theorem nttInvY_correct (s : State) (hs : (inPlaceK nttInv).pre s) :
+ ∃ t s', Exec isa nttInvAvx2 s t s' ∧ abiPreserved s s' ∧ (inPlaceK nttInv).post s s' := by
+ have hd : (pR (s.gpr .rsi)).Disjoint (pR (s.gpr .rdi)) := hs.2.2.1.symm
+ refine ymx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 =>
+ WP.mono (ynttBody_ok hs k1 f1
+ (G := (nttInvLens.foldl nttInvLayer (polyAt s.mem (s.gpr .rdi))).map (· * 8347681))
+ fun s2 hI hdi hsi hwf hw => ?_) fun s' ⟨hP, hf⟩ => ⟨by rw [nttInv_eq_layers]; exact hP, hf⟩
+ simp only [nttInvLens, List.foldl_cons, List.foldl_nil]
+ refine LIY.seq hdi hsi hwf hw hd (yinvLay1_ok hd) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay2_ok hd) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay4_ok hd) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 8 31 (by decide) (by decide)) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 16 15 (by decide) (by decide)) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 32 7 (by decide) (by decide)) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 64 3 (by decide) (by decide)) ?_ hI
+ refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 128 1 (by decide) (by decide)) ?_ hI
+ exact fun _ hI => LIY.seq hdi hsi hwf hw hd (yscale_ok hd) (fun _ hI => hI.epi) hI
+
+theorem nttY_ct : ConstantTime isa (inPlaceK ntt).pre (inPlaceK ntt).pub nttAvx2 :=
+ VG.Taint.constantTime (A := taint) (X86_64.Taint.ofRegs [.rdi, .rsi, .rsp]) inPlace_agree (by taint_decide)
+
+theorem nttInvY_ct : ConstantTime isa (inPlaceK nttInv).pre (inPlaceK nttInv).pub nttInvAvx2 :=
+ VG.Taint.constantTime (A := taint) (X86_64.Taint.ofRegs [.rdi, .rsi, .rsp]) inPlace_agree (by taint_decide)
+
+theorem nttY_verified : Verified X86_64.target nttAvx2 (Spec.MlDsa.nttContract X86_64.abi) :=
+ Verified.of_correct nttY_correct nttY_ct (by
+ mldsa_implies [Spec.MlDsa.nttContract, Spec.MlDsa.inPlaceContract, Spec.MlDsa.inPlaceSig, inPlaceK,
+ X86_64.abi, X86_64.argRegs] [inPlaceSat] using inPlaceSat)
+
+theorem nttInvY_verified : Verified X86_64.target nttInvAvx2 (Spec.MlDsa.nttInvContract X86_64.abi) :=
+ Verified.of_correct nttInvY_correct nttInvY_ct (by
+ mldsa_implies [Spec.MlDsa.nttInvContract, Spec.MlDsa.inPlaceContract, Spec.MlDsa.inPlaceSig, inPlaceK,
+ X86_64.abi, X86_64.argRegs] [inPlaceSat] using inPlaceSat)
+
+end VG.Proof.MlDsa.X86_64.Arith
diff --git a/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean
new file mode 100644
index 000000000..5b74ee5fe
--- /dev/null
+++ b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean
@@ -0,0 +1,18 @@
+import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.BackendAvx2
+
+/-!
+# ML-DSA's polynomial arithmetic on x86-64: AVX2
+
+A variant of `MlDsaArith` on x86-64 (see `TCB/Emit.lean`):
+`vg_mldsa_ntt_avx2`, `vg_mldsa_inv_ntt_avx2`,
+`vg_mldsa_multiply_ntt_avx2`, `vg_mldsa_multiply_add_ntt_avx2`,
+`vg_mldsa_add_avx2` and `vg_mldsa_sub_avx2`, on eight coefficients at a time
+in AVX2 registers, which need AVX and AVX2; key generation, signing and
+verification calling them need them too.
+-/
+
+namespace VG.Variants.MlDsaArith.X86_64.Avx2
+
+def variant : Proof.MlDsa.X86_64.ArithImpl := .avx2
+
+end VG.Variants.MlDsaArith.X86_64.Avx2
diff --git a/src/asm/x86_64/mldsa.rs b/src/asm/x86_64/mldsa.rs
index 01e804f36..541b9d25a 100644
--- a/src/asm/x86_64/mldsa.rs
+++ b/src/asm/x86_64/mldsa.rs
@@ -1754,6 +1754,1688 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_sub(f: *mut [u32; 256], g: *const
)
}
+/// The CPU features `vg_mldsa_ntt_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA_NTT_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// The ML-DSA number-theoretic transform, `NTT` (FIPS 204 Algorithm 41), of the polynomial `*f` (256 coefficients less than `q` = 8380417), in place.
+///
+/// Contract: `VG.Spec.MlDsa.nttContract`. Constant time: only the pointers may affect timing, not the data.
+///
+/// The function computes on eight coefficients at a time in AVX2 registers, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning.
+///
+/// # Safety
+///
+/// * `f` must be valid for reads and writes of 1024 bytes.
+/// * `scratch` must be valid for reads and writes of 1024 bytes.
+/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417.
+/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `f` and `scratch` must not overlap each other (distinct Rust objects never do).
+/// * Neither `f` nor `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa_ntt_avx2(f: *mut [u32; 256], scratch: *mut [u64; 128]) {
+ core::arch::naked_asm!(
+ "stmxcsr DWORD PTR [rsi+768]",
+ "mov r11d, DWORD PTR [rsi+768]",
+ "and r11d, 65535",
+ "mov eax, 8127",
+ "mov DWORD PTR [rsi+772], eax",
+ "ldmxcsr DWORD PTR [rsi+772]",
+ "lfence",
+ "movabs r9, 111012023893504",
+ "mov QWORD PTR [rsi], r9",
+ "movabs r9, 33764919763013891",
+ "mov QWORD PTR [rsi+8], r9",
+ "movabs r9, 32652304184483396",
+ "mov QWORD PTR [rsi+16], r9",
+ "movabs r9, 2003464812134697",
+ "mov QWORD PTR [rsi+24], r9",
+ "movabs r9, 10107995079564843",
+ "mov QWORD PTR [rsi+32], r9",
+ "movabs r9, 27008953388524718",
+ "mov QWORD PTR [rsi+40], r9",
+ "movabs r9, 23603259066260085",
+ "mov QWORD PTR [rsi+48], r9",
+ "movabs r9, 11510954738022985",
+ "mov QWORD PTR [rsi+56], r9",
+ "movabs r9, 4398527550166616",
+ "mov QWORD PTR [rsi+64], r9",
+ "movabs r9, 15401443493111205",
+ "mov QWORD PTR [rsi+72], r9",
+ "movabs r9, 31185040284548497",
+ "mov QWORD PTR [rsi+80], r9",
+ "movabs r9, 26937467947448680",
+ "mov QWORD PTR [rsi+88], r9",
+ "movabs r9, 19416172761943511",
+ "mov QWORD PTR [rsi+96], r9",
+ "movabs r9, 21916122901808376",
+ "mov QWORD PTR [rsi+104], r9",
+ "movabs r9, 35910200088776757",
+ "mov QWORD PTR [rsi+112], r9",
+ "movabs r9, 1202612321726977",
+ "mov QWORD PTR [rsi+120], r9",
+ "movabs r9, 411354790447719",
+ "mov QWORD PTR [rsi+128], r9",
+ "movabs r9, 15163111458665677",
+ "mov QWORD PTR [rsi+136], r9",
+ "movabs r9, 20565458766169348",
+ "mov QWORD PTR [rsi+144], r9",
+ "movabs r9, 16816682460325845",
+ "mov QWORD PTR [rsi+152], r9",
+ "movabs r9, 22920956268049798",
+ "mov QWORD PTR [rsi+160], r9",
+ "movabs r9, 23677166863944342",
+ "mov QWORD PTR [rsi+168], r9",
+ "movabs r9, 34703121006855168",
+ "mov QWORD PTR [rsi+176], r9",
+ "movabs r9, 33677345376425628",
+ "mov QWORD PTR [rsi+184], r9",
+ "movabs r9, 28933472397947454",
+ "mov QWORD PTR [rsi+192], r9",
+ "movabs r9, 19579390106369566",
+ "mov QWORD PTR [rsi+200], r9",
+ "movabs r9, 26799599498134591",
+ "mov QWORD PTR [rsi+208], r9",
+ "movabs r9, 15889643835192413",
+ "mov QWORD PTR [rsi+216], r9",
+ "movabs r9, 2282148053410728",
+ "mov QWORD PTR [rsi+224], r9",
+ "movabs r9, 16668952760323958",
+ "mov QWORD PTR [rsi+232], r9",
+ "movabs r9, 25011900965946676",
+ "mov QWORD PTR [rsi+240], r9",
+ "movabs r9, 23977247637478740",
+ "mov QWORD PTR [rsi+248], r9",
+ "movabs r9, 29427887555995366",
+ "mov QWORD PTR [rsi+256], r9",
+ "movabs r9, 22931526182748624",
+ "mov QWORD PTR [rsi+264], r9",
+ "movabs r9, 14929091579459166",
+ "mov QWORD PTR [rsi+272], r9",
+ "movabs r9, 29185144592086471",
+ "mov QWORD PTR [rsi+280], r9",
+ "movabs r9, 8329290213797750",
+ "mov QWORD PTR [rsi+288], r9",
+ "movabs r9, 31697782066745459",
+ "mov QWORD PTR [rsi+296], r9",
+ "movabs r9, 22432987854353025",
+ "mov QWORD PTR [rsi+304], r9",
+ "movabs r9, 545125843890401",
+ "mov QWORD PTR [rsi+312], r9",
+ "movabs r9, 31769864501989618",
+ "mov QWORD PTR [rsi+320], r9",
+ "movabs r9, 11662103226140216",
+ "mov QWORD PTR [rsi+328], r9",
+ "movabs r9, 20130185304250276",
+ "mov QWORD PTR [rsi+336], r9",
+ "movabs r9, 25354781094156910",
+ "mov QWORD PTR [rsi+344], r9",
+ "movabs r9, 30717142252233347",
+ "mov QWORD PTR [rsi+352], r9",
+ "movabs r9, 30375073877558844",
+ "mov QWORD PTR [rsi+360], r9",
+ "movabs r9, 5794237307816926",
+ "mov QWORD PTR [rsi+368], r9",
+ "movabs r9, 29849966874477923",
+ "mov QWORD PTR [rsi+376], r9",
+ "movabs r9, 1137925820308458",
+ "mov QWORD PTR [rsi+384], r9",
+ "movabs r9, 13305774323778583",
+ "mov QWORD PTR [rsi+392], r9",
+ "movabs r9, 31268732009491712",
+ "mov QWORD PTR [rsi+400], r9",
+ "movabs r9, 17002134848261444",
+ "mov QWORD PTR [rsi+408], r9",
+ "movabs r9, 35956774717033419",
+ "mov QWORD PTR [rsi+416], r9",
+ "movabs r9, 22036141462600008",
+ "mov QWORD PTR [rsi+424], r9",
+ "movabs r9, 35087477629023596",
+ "mov QWORD PTR [rsi+432], r9",
+ "movabs r9, 30337763489061025",
+ "mov QWORD PTR [rsi+440], r9",
+ "movabs r9, 20732429908239468",
+ "mov QWORD PTR [rsi+448], r9",
+ "movabs r9, 28041905903253186",
+ "mov QWORD PTR [rsi+456], r9",
+ "movabs r9, 35231517950811284",
+ "mov QWORD PTR [rsi+464], r9",
+ "movabs r9, 5760968484459269",
+ "mov QWORD PTR [rsi+472], r9",
+ "movabs r9, 29186403016613413",
+ "mov QWORD PTR [rsi+480], r9",
+ "movabs r9, 29809972144732485",
+ "mov QWORD PTR [rsi+488], r9",
+ "movabs r9, 19324591173389987",
+ "mov QWORD PTR [rsi+496], r9",
+ "movabs r9, 16492506917666904",
+ "mov QWORD PTR [rsi+504], r9",
+ "movabs r9, 14635985826474643",
+ "mov QWORD PTR [rsi+512], r9",
+ "movabs r9, 16398082059229396",
+ "mov QWORD PTR [rsi+520], r9",
+ "movabs r9, 9638297459285875",
+ "mov QWORD PTR [rsi+528], r9",
+ "movabs r9, 20692959164533664",
+ "mov QWORD PTR [rsi+536], r9",
+ "movabs r9, 10455835889373941",
+ "mov QWORD PTR [rsi+544], r9",
+ "movabs r9, 15088997507073265",
+ "mov QWORD PTR [rsi+552], r9",
+ "movabs r9, 19847271512942753",
+ "mov QWORD PTR [rsi+560], r9",
+ "movabs r9, 22278162875259735",
+ "mov QWORD PTR [rsi+568], r9",
+ "movabs r9, 7984765110959710",
+ "mov QWORD PTR [rsi+576], r9",
+ "movabs r9, 3517724245221606",
+ "mov QWORD PTR [rsi+584], r9",
+ "movabs r9, 29065087369580419",
+ "mov QWORD PTR [rsi+592], r9",
+ "movabs r9, 33749496538019589",
+ "mov QWORD PTR [rsi+600], r9",
+ "movabs r9, 22582830675910690",
+ "mov QWORD PTR [rsi+608], r9",
+ "movabs r9, 13774157688799364",
+ "mov QWORD PTR [rsi+616], r9",
+ "movabs r9, 33738338209470846",
+ "mov QWORD PTR [rsi+624], r9",
+ "movabs r9, 20549610337740179",
+ "mov QWORD PTR [rsi+632], r9",
+ "movabs r9, 1232604074686745",
+ "mov QWORD PTR [rsi+640], r9",
+ "movabs r9, 17645078572608834",
+ "mov QWORD PTR [rsi+648], r9",
+ "movabs r9, 21638646536106727",
+ "mov QWORD PTR [rsi+656], r9",
+ "movabs r9, 872067341384903",
+ "mov QWORD PTR [rsi+664], r9",
+ "movabs r9, 11559822875647717",
+ "mov QWORD PTR [rsi+672], r9",
+ "movabs r9, 5433172289935931",
+ "mov QWORD PTR [rsi+680], r9",
+ "movabs r9, 5358487101890844",
+ "mov QWORD PTR [rsi+688], r9",
+ "movabs r9, 6854656137752657",
+ "mov QWORD PTR [rsi+696], r9",
+ "movabs r9, 5370830838457625",
+ "mov QWORD PTR [rsi+704], r9",
+ "movabs r9, 20753904747165841",
+ "mov QWORD PTR [rsi+712], r9",
+ "movabs r9, 8027804982718602",
+ "mov QWORD PTR [rsi+720], r9",
+ "movabs r9, 31479735164668747",
+ "mov QWORD PTR [rsi+728], r9",
+ "movabs r9, 5314055668205759",
+ "mov QWORD PTR [rsi+736], r9",
+ "movabs r9, 29850847345983039",
+ "mov QWORD PTR [rsi+744], r9",
+ "movabs r9, 5636951310400997",
+ "mov QWORD PTR [rsi+752], r9",
+ "movabs r9, 27564133741392515",
+ "mov QWORD PTR [rsi+760], r9",
+ "movabs r9, 8233800205883732",
+ "mov QWORD PTR [rsi+768], r9",
+ "movabs r9, 30088883024233849",
+ "mov QWORD PTR [rsi+776], r9",
+ "movabs r9, 3338009929245701",
+ "mov QWORD PTR [rsi+784], r9",
+ "movabs r9, 14628791756398056",
+ "mov QWORD PTR [rsi+792], r9",
+ "movabs r9, 23830870862829877",
+ "mov QWORD PTR [rsi+800], r9",
+ "movabs r9, 28061014216302585",
+ "mov QWORD PTR [rsi+808], r9",
+ "movabs r9, 19997999096164636",
+ "mov QWORD PTR [rsi+816], r9",
+ "movabs r9, 19771555530215640",
+ "mov QWORD PTR [rsi+824], r9",
+ "movabs r9, 10447056982320729",
+ "mov QWORD PTR [rsi+832], r9",
+ "movabs r9, 35286145636332471",
+ "mov QWORD PTR [rsi+840], r9",
+ "movabs r9, 14470225858518424",
+ "mov QWORD PTR [rsi+848], r9",
+ "movabs r9, 30807937853347003",
+ "mov QWORD PTR [rsi+856], r9",
+ "movabs r9, 699409659546815",
+ "mov QWORD PTR [rsi+864], r9",
+ "movabs r9, 12945035726727688",
+ "mov QWORD PTR [rsi+872], r9",
+ "movabs r9, 7098008983067813",
+ "mov QWORD PTR [rsi+880], r9",
+ "movabs r9, 28266511219523944",
+ "mov QWORD PTR [rsi+888], r9",
+ "movabs r9, 15135022374814013",
+ "mov QWORD PTR [rsi+896], r9",
+ "movabs r9, 1158610381635861",
+ "mov QWORD PTR [rsi+904], r9",
+ "movabs r9, 31802227079365879",
+ "mov QWORD PTR [rsi+912], r9",
+ "movabs r9, 2027559572878823",
+ "mov QWORD PTR [rsi+920], r9",
+ "movabs r9, 7402805639339334",
+ "mov QWORD PTR [rsi+928], r9",
+ "movabs r9, 8205002449640623",
+ "mov QWORD PTR [rsi+936], r9",
+ "movabs r9, 31250542829661849",
+ "mov QWORD PTR [rsi+944], r9",
+ "movabs r9, 19527171898598875",
+ "mov QWORD PTR [rsi+952], r9",
+ "movabs r9, 26390134497937253",
+ "mov QWORD PTR [rsi+960], r9",
+ "movabs r9, 26173917256840158",
+ "mov QWORD PTR [rsi+968], r9",
+ "movabs r9, 31797902045269139",
+ "mov QWORD PTR [rsi+976], r9",
+ "movabs r9, 20765007236602922",
+ "mov QWORD PTR [rsi+984], r9",
+ "movabs r9, 16834811519265361",
+ "mov QWORD PTR [rsi+992], r9",
+ "movabs r9, 30142973844857679",
+ "mov QWORD PTR [rsi+1000], r9",
+ "movabs r9, 6014775284471242",
+ "mov QWORD PTR [rsi+1008], r9",
+ "movabs r9, 8490214048855735",
+ "mov QWORD PTR [rsi+1016], r9",
+ "mov eax, 8380417",
+ "vmovq xmm15, rax",
+ "vpbroadcastd ymm15, xmm15",
+ "mov eax, -58728449",
+ "vmovq xmm14, rax",
+ "vpbroadcastd ymm14, xmm14",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 4",
+ "mov eax, 1",
+ "20:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, 4",
+ "mov ecx, 16",
+ "21:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+512]",
+ "vpshufd ymm4, ymm1, 245",
+ "vpmuludq ymm1, ymm1, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm1, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm1, ymm1, ymm2",
+ "vpsrlq ymm1, ymm1, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm1, ymm1, ymm4",
+ "vpsubd ymm1, ymm1, ymm15",
+ "vpsrad ymm2, ymm1, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm1, ymm1, ymm2",
+ "vmovdqa ymm3, ymm0",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpsubd ymm3, ymm3, ymm1",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+512], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 21b",
+ "add rdx, 512",
+ "sub rax, 1",
+ "jne 20b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 8",
+ "mov eax, 2",
+ "22:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, 4",
+ "mov ecx, 8",
+ "23:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+256]",
+ "vpshufd ymm4, ymm1, 245",
+ "vpmuludq ymm1, ymm1, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm1, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm1, ymm1, ymm2",
+ "vpsrlq ymm1, ymm1, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm1, ymm1, ymm4",
+ "vpsubd ymm1, ymm1, ymm15",
+ "vpsrad ymm2, ymm1, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm1, ymm1, ymm2",
+ "vmovdqa ymm3, ymm0",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpsubd ymm3, ymm3, ymm1",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+256], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 23b",
+ "add rdx, 256",
+ "sub rax, 1",
+ "jne 22b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 16",
+ "mov eax, 4",
+ "24:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, 4",
+ "mov ecx, 4",
+ "25:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+128]",
+ "vpshufd ymm4, ymm1, 245",
+ "vpmuludq ymm1, ymm1, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm1, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm1, ymm1, ymm2",
+ "vpsrlq ymm1, ymm1, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm1, ymm1, ymm4",
+ "vpsubd ymm1, ymm1, ymm15",
+ "vpsrad ymm2, ymm1, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm1, ymm1, ymm2",
+ "vmovdqa ymm3, ymm0",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpsubd ymm3, ymm3, ymm1",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+128], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 25b",
+ "add rdx, 128",
+ "sub rax, 1",
+ "jne 24b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 32",
+ "mov eax, 8",
+ "26:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, 4",
+ "mov ecx, 2",
+ "27:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+64]",
+ "vpshufd ymm4, ymm1, 245",
+ "vpmuludq ymm1, ymm1, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm1, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm1, ymm1, ymm2",
+ "vpsrlq ymm1, ymm1, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm1, ymm1, ymm4",
+ "vpsubd ymm1, ymm1, ymm15",
+ "vpsrad ymm2, ymm1, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm1, ymm1, ymm2",
+ "vmovdqa ymm3, ymm0",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpsubd ymm3, ymm3, ymm1",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+64], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 27b",
+ "add rdx, 64",
+ "sub rax, 1",
+ "jne 26b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 64",
+ "mov eax, 16",
+ "28:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, 4",
+ "mov ecx, 1",
+ "29:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+32]",
+ "vpshufd ymm4, ymm1, 245",
+ "vpmuludq ymm1, ymm1, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm1, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm1, ymm1, ymm2",
+ "vpsrlq ymm1, ymm1, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm1, ymm1, ymm4",
+ "vpsubd ymm1, ymm1, ymm15",
+ "vpsrad ymm2, ymm1, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm1, ymm1, ymm2",
+ "vmovdqa ymm3, ymm0",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpsubd ymm3, ymm3, ymm1",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+32], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 29b",
+ "add rdx, 32",
+ "sub rax, 1",
+ "jne 28b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 128",
+ "mov ecx, 16",
+ "210:",
+ "vmovdqu ymm4, YMMWORD PTR [rdx]",
+ "vmovdqu ymm5, YMMWORD PTR [rdx+32]",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm2, ymm13, 85",
+ "vpshufd ymm13, ymm13, 0",
+ "vpblendd ymm13, ymm13, ymm2, 240",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, 8",
+ "vperm2i128 ymm0, ymm4, ymm5, 32",
+ "vperm2i128 ymm1, ymm4, ymm5, 49",
+ "vpshufd ymm4, ymm1, 245",
+ "vpmuludq ymm1, ymm1, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm1, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm1, ymm1, ymm2",
+ "vpsrlq ymm1, ymm1, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm1, ymm1, ymm4",
+ "vpsubd ymm1, ymm1, ymm15",
+ "vpsrad ymm2, ymm1, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm1, ymm1, ymm2",
+ "vmovdqa ymm3, ymm0",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpsubd ymm3, ymm3, ymm1",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vperm2i128 ymm4, ymm0, ymm3, 32",
+ "vperm2i128 ymm5, ymm0, ymm3, 49",
+ "vmovdqu YMMWORD PTR [rdx], ymm4",
+ "vmovdqu YMMWORD PTR [rdx+32], ymm5",
+ "add rdx, 64",
+ "sub rcx, 1",
+ "jne 210b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 256",
+ "mov ecx, 16",
+ "211:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+32]",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm2, ymm13, 245",
+ "vpshufd ymm13, ymm13, 160",
+ "vpblendd ymm13, ymm13, ymm2, 240",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, 16",
+ "vmovdqa ymm2, ymm0",
+ "vpunpcklqdq ymm0, ymm0, ymm1",
+ "vpunpckhqdq ymm2, ymm2, ymm1",
+ "vmovdqa ymm1, ymm2",
+ "vpshufd ymm4, ymm1, 245",
+ "vpmuludq ymm1, ymm1, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm1, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm1, ymm1, ymm2",
+ "vpsrlq ymm1, ymm1, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm1, ymm1, ymm4",
+ "vpsubd ymm1, ymm1, ymm15",
+ "vpsrad ymm2, ymm1, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm1, ymm1, ymm2",
+ "vmovdqa ymm3, ymm0",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpsubd ymm3, ymm3, ymm1",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqa ymm1, ymm0",
+ "vpunpcklqdq ymm0, ymm0, ymm3",
+ "vpunpckhqdq ymm1, ymm1, ymm3",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+32], ymm1",
+ "add rdx, 64",
+ "sub rcx, 1",
+ "jne 211b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 512",
+ "mov ecx, 16",
+ "212:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm2, YMMWORD PTR [rdx+32]",
+ "vmovdqu ymm13, YMMWORD PTR [r8]",
+ "vpermq ymm13, ymm13, 216",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, 32",
+ "vpshufd ymm0, ymm0, 216",
+ "vpshufd ymm2, ymm2, 216",
+ "vmovdqa ymm1, ymm0",
+ "vpunpcklqdq ymm0, ymm0, ymm2",
+ "vpunpckhqdq ymm1, ymm1, ymm2",
+ "vpshufd ymm4, ymm1, 245",
+ "vpmuludq ymm1, ymm1, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm1, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm1, ymm1, ymm2",
+ "vpsrlq ymm1, ymm1, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm1, ymm1, ymm4",
+ "vpsubd ymm1, ymm1, ymm15",
+ "vpsrad ymm2, ymm1, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm1, ymm1, ymm2",
+ "vmovdqa ymm3, ymm0",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpsubd ymm3, ymm3, ymm1",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqa ymm1, ymm0",
+ "vpunpckldq ymm0, ymm0, ymm3",
+ "vpunpckhdq ymm1, ymm1, ymm3",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+32], ymm1",
+ "add rdx, 64",
+ "sub rcx, 1",
+ "jne 212b",
+ "vzeroupper",
+ "lfence",
+ "mov DWORD PTR [rsi+768], r11d",
+ "ldmxcsr DWORD PTR [rsi+768]",
+ "ret",
+ )
+}
+
+/// The CPU features `vg_mldsa_inv_ntt_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA_INV_NTT_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// The inverse of the ML-DSA number-theoretic transform, `NTT⁻¹` (FIPS 204 Algorithm 42), of `*f` (256 coefficients less than `q` = 8380417), in place.
+///
+/// Contract: `VG.Spec.MlDsa.nttInvContract`. Constant time: only the pointers may affect timing, not the data.
+///
+/// The function computes on eight coefficients at a time in AVX2 registers, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning.
+///
+/// # Safety
+///
+/// * `f` must be valid for reads and writes of 1024 bytes.
+/// * `scratch` must be valid for reads and writes of 1024 bytes.
+/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417.
+/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `f` and `scratch` must not overlap each other (distinct Rust objects never do).
+/// * Neither `f` nor `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa_inv_ntt_avx2(f: *mut [u32; 256], scratch: *mut [u64; 128]) {
+ core::arch::naked_asm!(
+ "stmxcsr DWORD PTR [rsi+768]",
+ "mov r11d, DWORD PTR [rsi+768]",
+ "and r11d, 65535",
+ "mov eax, 8127",
+ "mov DWORD PTR [rsi+772], eax",
+ "ldmxcsr DWORD PTR [rsi+772]",
+ "lfence",
+ "movabs r9, 111012023893504",
+ "mov QWORD PTR [rsi], r9",
+ "movabs r9, 33764919763013891",
+ "mov QWORD PTR [rsi+8], r9",
+ "movabs r9, 32652304184483396",
+ "mov QWORD PTR [rsi+16], r9",
+ "movabs r9, 2003464812134697",
+ "mov QWORD PTR [rsi+24], r9",
+ "movabs r9, 10107995079564843",
+ "mov QWORD PTR [rsi+32], r9",
+ "movabs r9, 27008953388524718",
+ "mov QWORD PTR [rsi+40], r9",
+ "movabs r9, 23603259066260085",
+ "mov QWORD PTR [rsi+48], r9",
+ "movabs r9, 11510954738022985",
+ "mov QWORD PTR [rsi+56], r9",
+ "movabs r9, 4398527550166616",
+ "mov QWORD PTR [rsi+64], r9",
+ "movabs r9, 15401443493111205",
+ "mov QWORD PTR [rsi+72], r9",
+ "movabs r9, 31185040284548497",
+ "mov QWORD PTR [rsi+80], r9",
+ "movabs r9, 26937467947448680",
+ "mov QWORD PTR [rsi+88], r9",
+ "movabs r9, 19416172761943511",
+ "mov QWORD PTR [rsi+96], r9",
+ "movabs r9, 21916122901808376",
+ "mov QWORD PTR [rsi+104], r9",
+ "movabs r9, 35910200088776757",
+ "mov QWORD PTR [rsi+112], r9",
+ "movabs r9, 1202612321726977",
+ "mov QWORD PTR [rsi+120], r9",
+ "movabs r9, 411354790447719",
+ "mov QWORD PTR [rsi+128], r9",
+ "movabs r9, 15163111458665677",
+ "mov QWORD PTR [rsi+136], r9",
+ "movabs r9, 20565458766169348",
+ "mov QWORD PTR [rsi+144], r9",
+ "movabs r9, 16816682460325845",
+ "mov QWORD PTR [rsi+152], r9",
+ "movabs r9, 22920956268049798",
+ "mov QWORD PTR [rsi+160], r9",
+ "movabs r9, 23677166863944342",
+ "mov QWORD PTR [rsi+168], r9",
+ "movabs r9, 34703121006855168",
+ "mov QWORD PTR [rsi+176], r9",
+ "movabs r9, 33677345376425628",
+ "mov QWORD PTR [rsi+184], r9",
+ "movabs r9, 28933472397947454",
+ "mov QWORD PTR [rsi+192], r9",
+ "movabs r9, 19579390106369566",
+ "mov QWORD PTR [rsi+200], r9",
+ "movabs r9, 26799599498134591",
+ "mov QWORD PTR [rsi+208], r9",
+ "movabs r9, 15889643835192413",
+ "mov QWORD PTR [rsi+216], r9",
+ "movabs r9, 2282148053410728",
+ "mov QWORD PTR [rsi+224], r9",
+ "movabs r9, 16668952760323958",
+ "mov QWORD PTR [rsi+232], r9",
+ "movabs r9, 25011900965946676",
+ "mov QWORD PTR [rsi+240], r9",
+ "movabs r9, 23977247637478740",
+ "mov QWORD PTR [rsi+248], r9",
+ "movabs r9, 29427887555995366",
+ "mov QWORD PTR [rsi+256], r9",
+ "movabs r9, 22931526182748624",
+ "mov QWORD PTR [rsi+264], r9",
+ "movabs r9, 14929091579459166",
+ "mov QWORD PTR [rsi+272], r9",
+ "movabs r9, 29185144592086471",
+ "mov QWORD PTR [rsi+280], r9",
+ "movabs r9, 8329290213797750",
+ "mov QWORD PTR [rsi+288], r9",
+ "movabs r9, 31697782066745459",
+ "mov QWORD PTR [rsi+296], r9",
+ "movabs r9, 22432987854353025",
+ "mov QWORD PTR [rsi+304], r9",
+ "movabs r9, 545125843890401",
+ "mov QWORD PTR [rsi+312], r9",
+ "movabs r9, 31769864501989618",
+ "mov QWORD PTR [rsi+320], r9",
+ "movabs r9, 11662103226140216",
+ "mov QWORD PTR [rsi+328], r9",
+ "movabs r9, 20130185304250276",
+ "mov QWORD PTR [rsi+336], r9",
+ "movabs r9, 25354781094156910",
+ "mov QWORD PTR [rsi+344], r9",
+ "movabs r9, 30717142252233347",
+ "mov QWORD PTR [rsi+352], r9",
+ "movabs r9, 30375073877558844",
+ "mov QWORD PTR [rsi+360], r9",
+ "movabs r9, 5794237307816926",
+ "mov QWORD PTR [rsi+368], r9",
+ "movabs r9, 29849966874477923",
+ "mov QWORD PTR [rsi+376], r9",
+ "movabs r9, 1137925820308458",
+ "mov QWORD PTR [rsi+384], r9",
+ "movabs r9, 13305774323778583",
+ "mov QWORD PTR [rsi+392], r9",
+ "movabs r9, 31268732009491712",
+ "mov QWORD PTR [rsi+400], r9",
+ "movabs r9, 17002134848261444",
+ "mov QWORD PTR [rsi+408], r9",
+ "movabs r9, 35956774717033419",
+ "mov QWORD PTR [rsi+416], r9",
+ "movabs r9, 22036141462600008",
+ "mov QWORD PTR [rsi+424], r9",
+ "movabs r9, 35087477629023596",
+ "mov QWORD PTR [rsi+432], r9",
+ "movabs r9, 30337763489061025",
+ "mov QWORD PTR [rsi+440], r9",
+ "movabs r9, 20732429908239468",
+ "mov QWORD PTR [rsi+448], r9",
+ "movabs r9, 28041905903253186",
+ "mov QWORD PTR [rsi+456], r9",
+ "movabs r9, 35231517950811284",
+ "mov QWORD PTR [rsi+464], r9",
+ "movabs r9, 5760968484459269",
+ "mov QWORD PTR [rsi+472], r9",
+ "movabs r9, 29186403016613413",
+ "mov QWORD PTR [rsi+480], r9",
+ "movabs r9, 29809972144732485",
+ "mov QWORD PTR [rsi+488], r9",
+ "movabs r9, 19324591173389987",
+ "mov QWORD PTR [rsi+496], r9",
+ "movabs r9, 16492506917666904",
+ "mov QWORD PTR [rsi+504], r9",
+ "movabs r9, 14635985826474643",
+ "mov QWORD PTR [rsi+512], r9",
+ "movabs r9, 16398082059229396",
+ "mov QWORD PTR [rsi+520], r9",
+ "movabs r9, 9638297459285875",
+ "mov QWORD PTR [rsi+528], r9",
+ "movabs r9, 20692959164533664",
+ "mov QWORD PTR [rsi+536], r9",
+ "movabs r9, 10455835889373941",
+ "mov QWORD PTR [rsi+544], r9",
+ "movabs r9, 15088997507073265",
+ "mov QWORD PTR [rsi+552], r9",
+ "movabs r9, 19847271512942753",
+ "mov QWORD PTR [rsi+560], r9",
+ "movabs r9, 22278162875259735",
+ "mov QWORD PTR [rsi+568], r9",
+ "movabs r9, 7984765110959710",
+ "mov QWORD PTR [rsi+576], r9",
+ "movabs r9, 3517724245221606",
+ "mov QWORD PTR [rsi+584], r9",
+ "movabs r9, 29065087369580419",
+ "mov QWORD PTR [rsi+592], r9",
+ "movabs r9, 33749496538019589",
+ "mov QWORD PTR [rsi+600], r9",
+ "movabs r9, 22582830675910690",
+ "mov QWORD PTR [rsi+608], r9",
+ "movabs r9, 13774157688799364",
+ "mov QWORD PTR [rsi+616], r9",
+ "movabs r9, 33738338209470846",
+ "mov QWORD PTR [rsi+624], r9",
+ "movabs r9, 20549610337740179",
+ "mov QWORD PTR [rsi+632], r9",
+ "movabs r9, 1232604074686745",
+ "mov QWORD PTR [rsi+640], r9",
+ "movabs r9, 17645078572608834",
+ "mov QWORD PTR [rsi+648], r9",
+ "movabs r9, 21638646536106727",
+ "mov QWORD PTR [rsi+656], r9",
+ "movabs r9, 872067341384903",
+ "mov QWORD PTR [rsi+664], r9",
+ "movabs r9, 11559822875647717",
+ "mov QWORD PTR [rsi+672], r9",
+ "movabs r9, 5433172289935931",
+ "mov QWORD PTR [rsi+680], r9",
+ "movabs r9, 5358487101890844",
+ "mov QWORD PTR [rsi+688], r9",
+ "movabs r9, 6854656137752657",
+ "mov QWORD PTR [rsi+696], r9",
+ "movabs r9, 5370830838457625",
+ "mov QWORD PTR [rsi+704], r9",
+ "movabs r9, 20753904747165841",
+ "mov QWORD PTR [rsi+712], r9",
+ "movabs r9, 8027804982718602",
+ "mov QWORD PTR [rsi+720], r9",
+ "movabs r9, 31479735164668747",
+ "mov QWORD PTR [rsi+728], r9",
+ "movabs r9, 5314055668205759",
+ "mov QWORD PTR [rsi+736], r9",
+ "movabs r9, 29850847345983039",
+ "mov QWORD PTR [rsi+744], r9",
+ "movabs r9, 5636951310400997",
+ "mov QWORD PTR [rsi+752], r9",
+ "movabs r9, 27564133741392515",
+ "mov QWORD PTR [rsi+760], r9",
+ "movabs r9, 8233800205883732",
+ "mov QWORD PTR [rsi+768], r9",
+ "movabs r9, 30088883024233849",
+ "mov QWORD PTR [rsi+776], r9",
+ "movabs r9, 3338009929245701",
+ "mov QWORD PTR [rsi+784], r9",
+ "movabs r9, 14628791756398056",
+ "mov QWORD PTR [rsi+792], r9",
+ "movabs r9, 23830870862829877",
+ "mov QWORD PTR [rsi+800], r9",
+ "movabs r9, 28061014216302585",
+ "mov QWORD PTR [rsi+808], r9",
+ "movabs r9, 19997999096164636",
+ "mov QWORD PTR [rsi+816], r9",
+ "movabs r9, 19771555530215640",
+ "mov QWORD PTR [rsi+824], r9",
+ "movabs r9, 10447056982320729",
+ "mov QWORD PTR [rsi+832], r9",
+ "movabs r9, 35286145636332471",
+ "mov QWORD PTR [rsi+840], r9",
+ "movabs r9, 14470225858518424",
+ "mov QWORD PTR [rsi+848], r9",
+ "movabs r9, 30807937853347003",
+ "mov QWORD PTR [rsi+856], r9",
+ "movabs r9, 699409659546815",
+ "mov QWORD PTR [rsi+864], r9",
+ "movabs r9, 12945035726727688",
+ "mov QWORD PTR [rsi+872], r9",
+ "movabs r9, 7098008983067813",
+ "mov QWORD PTR [rsi+880], r9",
+ "movabs r9, 28266511219523944",
+ "mov QWORD PTR [rsi+888], r9",
+ "movabs r9, 15135022374814013",
+ "mov QWORD PTR [rsi+896], r9",
+ "movabs r9, 1158610381635861",
+ "mov QWORD PTR [rsi+904], r9",
+ "movabs r9, 31802227079365879",
+ "mov QWORD PTR [rsi+912], r9",
+ "movabs r9, 2027559572878823",
+ "mov QWORD PTR [rsi+920], r9",
+ "movabs r9, 7402805639339334",
+ "mov QWORD PTR [rsi+928], r9",
+ "movabs r9, 8205002449640623",
+ "mov QWORD PTR [rsi+936], r9",
+ "movabs r9, 31250542829661849",
+ "mov QWORD PTR [rsi+944], r9",
+ "movabs r9, 19527171898598875",
+ "mov QWORD PTR [rsi+952], r9",
+ "movabs r9, 26390134497937253",
+ "mov QWORD PTR [rsi+960], r9",
+ "movabs r9, 26173917256840158",
+ "mov QWORD PTR [rsi+968], r9",
+ "movabs r9, 31797902045269139",
+ "mov QWORD PTR [rsi+976], r9",
+ "movabs r9, 20765007236602922",
+ "mov QWORD PTR [rsi+984], r9",
+ "movabs r9, 16834811519265361",
+ "mov QWORD PTR [rsi+992], r9",
+ "movabs r9, 30142973844857679",
+ "mov QWORD PTR [rsi+1000], r9",
+ "movabs r9, 6014775284471242",
+ "mov QWORD PTR [rsi+1008], r9",
+ "movabs r9, 8490214048855735",
+ "mov QWORD PTR [rsi+1016], r9",
+ "mov eax, 8380417",
+ "vmovq xmm15, rax",
+ "vpbroadcastd ymm15, xmm15",
+ "mov eax, -58728449",
+ "vmovq xmm14, rax",
+ "vpbroadcastd ymm14, xmm14",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 992",
+ "mov ecx, 16",
+ "20:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm2, YMMWORD PTR [rdx+32]",
+ "vmovdqu ymm13, YMMWORD PTR [r8]",
+ "vpermq ymm13, ymm13, 39",
+ "vpshufd ymm13, ymm13, 177",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, -32",
+ "vpshufd ymm0, ymm0, 216",
+ "vpshufd ymm2, ymm2, 216",
+ "vmovdqa ymm1, ymm0",
+ "vpunpcklqdq ymm0, ymm0, ymm2",
+ "vpunpckhqdq ymm1, ymm1, ymm2",
+ "vpsubd ymm3, ymm1, ymm0",
+ "vpaddd ymm3, ymm3, ymm15",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqa ymm1, ymm0",
+ "vpunpckldq ymm0, ymm0, ymm3",
+ "vpunpckhdq ymm1, ymm1, ymm3",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+32], ymm1",
+ "add rdx, 64",
+ "sub rcx, 1",
+ "jne 20b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 496",
+ "mov ecx, 16",
+ "21:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+32]",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm2, ymm13, 10",
+ "vpshufd ymm13, ymm13, 95",
+ "vpblendd ymm13, ymm13, ymm2, 240",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, -16",
+ "vmovdqa ymm2, ymm0",
+ "vpunpcklqdq ymm0, ymm0, ymm1",
+ "vpunpckhqdq ymm2, ymm2, ymm1",
+ "vmovdqa ymm1, ymm2",
+ "vpsubd ymm3, ymm1, ymm0",
+ "vpaddd ymm3, ymm3, ymm15",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqa ymm1, ymm0",
+ "vpunpcklqdq ymm0, ymm0, ymm3",
+ "vpunpckhqdq ymm1, ymm1, ymm3",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+32], ymm1",
+ "add rdx, 64",
+ "sub rcx, 1",
+ "jne 21b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 248",
+ "mov ecx, 16",
+ "22:",
+ "vmovdqu ymm4, YMMWORD PTR [rdx]",
+ "vmovdqu ymm5, YMMWORD PTR [rdx+32]",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm2, ymm13, 0",
+ "vpshufd ymm13, ymm13, 85",
+ "vpblendd ymm13, ymm13, ymm2, 240",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, -8",
+ "vperm2i128 ymm0, ymm4, ymm5, 32",
+ "vperm2i128 ymm1, ymm4, ymm5, 49",
+ "vpsubd ymm3, ymm1, ymm0",
+ "vpaddd ymm3, ymm3, ymm15",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vperm2i128 ymm4, ymm0, ymm3, 32",
+ "vperm2i128 ymm5, ymm0, ymm3, 49",
+ "vmovdqu YMMWORD PTR [rdx], ymm4",
+ "vmovdqu YMMWORD PTR [rdx+32], ymm5",
+ "add rdx, 64",
+ "sub rcx, 1",
+ "jne 22b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 124",
+ "mov eax, 16",
+ "23:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, -4",
+ "mov ecx, 1",
+ "24:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+32]",
+ "vpsubd ymm3, ymm1, ymm0",
+ "vpaddd ymm3, ymm3, ymm15",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+32], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 24b",
+ "add rdx, 32",
+ "sub rax, 1",
+ "jne 23b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 60",
+ "mov eax, 8",
+ "25:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, -4",
+ "mov ecx, 2",
+ "26:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+64]",
+ "vpsubd ymm3, ymm1, ymm0",
+ "vpaddd ymm3, ymm3, ymm15",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+64], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 26b",
+ "add rdx, 64",
+ "sub rax, 1",
+ "jne 25b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 28",
+ "mov eax, 4",
+ "27:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, -4",
+ "mov ecx, 4",
+ "28:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+128]",
+ "vpsubd ymm3, ymm1, ymm0",
+ "vpaddd ymm3, ymm3, ymm15",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+128], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 28b",
+ "add rdx, 128",
+ "sub rax, 1",
+ "jne 27b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 12",
+ "mov eax, 2",
+ "29:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, -4",
+ "mov ecx, 8",
+ "210:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+256]",
+ "vpsubd ymm3, ymm1, ymm0",
+ "vpaddd ymm3, ymm3, ymm15",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+256], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 210b",
+ "add rdx, 256",
+ "sub rax, 1",
+ "jne 29b",
+ "mov rdx, rdi",
+ "mov r8, rsi",
+ "add r8, 4",
+ "mov eax, 1",
+ "211:",
+ "vbroadcasti128 ymm13, XMMWORD PTR [r8]",
+ "vpshufd ymm13, ymm13, 0",
+ "vpshufd ymm12, ymm13, 245",
+ "add r8, -4",
+ "mov ecx, 16",
+ "212:",
+ "vmovdqu ymm0, YMMWORD PTR [rdx]",
+ "vmovdqu ymm1, YMMWORD PTR [rdx+512]",
+ "vpsubd ymm3, ymm1, ymm0",
+ "vpaddd ymm3, ymm3, ymm15",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm0",
+ "vmovdqu YMMWORD PTR [rdx+512], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 212b",
+ "add rdx, 512",
+ "sub rax, 1",
+ "jne 211b",
+ "mov rdx, rdi",
+ "mov eax, 16382",
+ "vmovq xmm13, rax",
+ "vpbroadcastd ymm13, xmm13",
+ "vmovdqa ymm12, ymm13",
+ "mov ecx, 32",
+ "213:",
+ "vmovdqu ymm3, YMMWORD PTR [rdx]",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdx], ymm3",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 213b",
+ "vzeroupper",
+ "lfence",
+ "mov DWORD PTR [rsi+768], r11d",
+ "ldmxcsr DWORD PTR [rsi+768]",
+ "ret",
+ )
+}
+
+/// The CPU features `vg_mldsa_multiply_ntt_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA_MULTIPLY_NTT_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// The product of two elements of `T_q`, `MultiplyNTT` (FIPS 204 Algorithm 45): writes the coefficientwise product of `*f` and `*g` modulo `q` = 8380417 to `*h`.
+///
+/// Contract: `VG.Spec.MlDsa.mulContract`. Constant time: only the pointers may affect timing, not the data.
+///
+/// The function computes on eight coefficients at a time in AVX2 registers. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before returning.
+///
+/// # Safety
+///
+/// * `h` must be valid for reads and writes of 1024 bytes.
+/// * `f` must be valid for reads of 1024 bytes.
+/// * `g` must be valid for reads of 1024 bytes.
+/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417.
+/// * Each of the 256 `u32`s of `g` must be less than `q` = 8380417.
+/// * `h` must not overlap `f` or `g` (distinct Rust objects never do).
+/// * None of `h`, `f` and `g` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_ntt_avx2(h: *mut [u32; 256], f: *const [u32; 256], g: *const [u32; 256]) {
+ core::arch::naked_asm!(
+ "mov r8, rdi",
+ "vmovdqu ymm6, YMMWORD PTR [rdi+992]",
+ "stmxcsr DWORD PTR [r8+1016]",
+ "mov r11d, DWORD PTR [r8+1016]",
+ "and r11d, 65535",
+ "mov eax, 8127",
+ "mov DWORD PTR [r8+1020], eax",
+ "ldmxcsr DWORD PTR [r8+1020]",
+ "lfence",
+ "mov eax, 8380417",
+ "vmovq xmm15, rax",
+ "vpbroadcastd ymm15, xmm15",
+ "mov eax, -58728449",
+ "vmovq xmm14, rax",
+ "vpbroadcastd ymm14, xmm14",
+ "mov eax, 2365951",
+ "vmovq xmm11, rax",
+ "vpbroadcastd ymm11, xmm11",
+ "mov ecx, 31",
+ "20:",
+ "vmovdqu ymm3, YMMWORD PTR [rsi]",
+ "vmovdqu ymm13, YMMWORD PTR [rdx]",
+ "vmovdqu ymm5, YMMWORD PTR [rdi]",
+ "vpshufd ymm12, ymm13, 245",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm11",
+ "vpmuludq ymm4, ymm4, ymm11",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdi], ymm3",
+ "add rdi, 32",
+ "add rsi, 32",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 20b",
+ "vmovdqu ymm3, YMMWORD PTR [rsi]",
+ "vmovdqu ymm13, YMMWORD PTR [rdx]",
+ "vmovdqa ymm5, ymm6",
+ "vpshufd ymm12, ymm13, 245",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm11",
+ "vpmuludq ymm4, ymm4, ymm11",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "lfence",
+ "mov DWORD PTR [r8+1016], r11d",
+ "ldmxcsr DWORD PTR [r8+1016]",
+ "vmovdqu YMMWORD PTR [rdi], ymm3",
+ "vzeroupper",
+ "ret",
+ )
+}
+
+/// The CPU features `vg_mldsa_multiply_add_ntt_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA_MULTIPLY_ADD_NTT_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// Adds the product of two elements of `T_q` to a third, `AddNTT(h, MultiplyNTT(f, g))` (FIPS 204 Algorithms 44 and 45): adds the coefficientwise product of `*f` and `*g` to `*h`, modulo `q` = 8380417.
+///
+/// Contract: `VG.Spec.MlDsa.mulAddContract`. Constant time: only the pointers may affect timing, not the data.
+///
+/// The function computes on eight coefficients at a time in AVX2 registers. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before returning.
+///
+/// # Safety
+///
+/// * `h` must be valid for reads and writes of 1024 bytes.
+/// * `f` must be valid for reads of 1024 bytes.
+/// * `g` must be valid for reads of 1024 bytes.
+/// * Each of the 256 `u32`s of `h` must be less than `q` = 8380417.
+/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417.
+/// * Each of the 256 `u32`s of `g` must be less than `q` = 8380417.
+/// * `h` must not overlap `f` or `g` (distinct Rust objects never do).
+/// * None of `h`, `f` and `g` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_add_ntt_avx2(h: *mut [u32; 256], f: *const [u32; 256], g: *const [u32; 256]) {
+ core::arch::naked_asm!(
+ "mov r8, rdi",
+ "vmovdqu ymm6, YMMWORD PTR [rdi+992]",
+ "stmxcsr DWORD PTR [r8+1016]",
+ "mov r11d, DWORD PTR [r8+1016]",
+ "and r11d, 65535",
+ "mov eax, 8127",
+ "mov DWORD PTR [r8+1020], eax",
+ "ldmxcsr DWORD PTR [r8+1020]",
+ "lfence",
+ "mov eax, 8380417",
+ "vmovq xmm15, rax",
+ "vpbroadcastd ymm15, xmm15",
+ "mov eax, -58728449",
+ "vmovq xmm14, rax",
+ "vpbroadcastd ymm14, xmm14",
+ "mov eax, 2365951",
+ "vmovq xmm11, rax",
+ "vpbroadcastd ymm11, xmm11",
+ "mov ecx, 31",
+ "20:",
+ "vmovdqu ymm3, YMMWORD PTR [rsi]",
+ "vmovdqu ymm13, YMMWORD PTR [rdx]",
+ "vmovdqu ymm5, YMMWORD PTR [rdi]",
+ "vpshufd ymm12, ymm13, 245",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm11",
+ "vpmuludq ymm4, ymm4, ymm11",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vpaddd ymm3, ymm3, ymm5",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vmovdqu YMMWORD PTR [rdi], ymm3",
+ "add rdi, 32",
+ "add rsi, 32",
+ "add rdx, 32",
+ "sub rcx, 1",
+ "jne 20b",
+ "vmovdqu ymm3, YMMWORD PTR [rsi]",
+ "vmovdqu ymm13, YMMWORD PTR [rdx]",
+ "vmovdqa ymm5, ymm6",
+ "vpshufd ymm12, ymm13, 245",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm13",
+ "vpmuludq ymm4, ymm4, ymm12",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpshufd ymm4, ymm3, 245",
+ "vpmuludq ymm3, ymm3, ymm11",
+ "vpmuludq ymm4, ymm4, ymm11",
+ "vpmuludq ymm2, ymm3, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm3, ymm3, ymm2",
+ "vpsrlq ymm3, ymm3, 32",
+ "vpmuludq ymm2, ymm4, ymm14",
+ "vpmuludq ymm2, ymm2, ymm15",
+ "vpaddq ymm4, ymm4, ymm2",
+ "vpor ymm3, ymm3, ymm4",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "vpaddd ymm3, ymm3, ymm5",
+ "vpsubd ymm3, ymm3, ymm15",
+ "vpsrad ymm2, ymm3, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm3, ymm3, ymm2",
+ "lfence",
+ "mov DWORD PTR [r8+1016], r11d",
+ "ldmxcsr DWORD PTR [r8+1016]",
+ "vmovdqu YMMWORD PTR [rdi], ymm3",
+ "vzeroupper",
+ "ret",
+ )
+}
+
+/// The CPU features `vg_mldsa_add_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA_ADD_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// Adds the polynomial `*g` to `*f` modulo `q` = 8380417, coefficient by coefficient (FIPS 204 Algorithm 44).
+///
+/// Contract: `VG.Spec.MlDsa.addContract`. Constant time: only the pointers may affect timing, not the data.
+///
+/// The function computes on eight coefficients at a time in AVX2 registers.
+///
+/// # Safety
+///
+/// * `f` must be valid for reads and writes of 1024 bytes.
+/// * `g` must be valid for reads of 1024 bytes.
+/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417.
+/// * Each of the 256 `u32`s of `g` must be less than `q` = 8380417.
+/// * `f` must not overlap `g` (distinct Rust objects never do).
+/// * Neither `f` nor `g` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa_add_avx2(f: *mut [u32; 256], g: *const [u32; 256]) {
+ core::arch::naked_asm!(
+ "mov eax, 8380417",
+ "vmovq xmm15, rax",
+ "vpbroadcastd ymm15, xmm15",
+ "mov ecx, 32",
+ "20:",
+ "vmovdqu ymm0, YMMWORD PTR [rdi]",
+ "vmovdqu ymm1, YMMWORD PTR [rsi]",
+ "vpaddd ymm0, ymm0, ymm1",
+ "vpsubd ymm0, ymm0, ymm15",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vmovdqu YMMWORD PTR [rdi], ymm0",
+ "add rdi, 32",
+ "add rsi, 32",
+ "sub rcx, 1",
+ "jne 20b",
+ "vzeroupper",
+ "ret",
+ )
+}
+
+/// The CPU features `vg_mldsa_sub_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA_SUB_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// Subtracts the polynomial `*g` from `*f` modulo `q` = 8380417, coefficient by coefficient.
+///
+/// Contract: `VG.Spec.MlDsa.subContract`. Constant time: only the pointers may affect timing, not the data.
+///
+/// The function computes on eight coefficients at a time in AVX2 registers.
+///
+/// # Safety
+///
+/// * `f` must be valid for reads and writes of 1024 bytes.
+/// * `g` must be valid for reads of 1024 bytes.
+/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417.
+/// * Each of the 256 `u32`s of `g` must be less than `q` = 8380417.
+/// * `f` must not overlap `g` (distinct Rust objects never do).
+/// * Neither `f` nor `g` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa_sub_avx2(f: *mut [u32; 256], g: *const [u32; 256]) {
+ core::arch::naked_asm!(
+ "mov eax, 8380417",
+ "vmovq xmm15, rax",
+ "vpbroadcastd ymm15, xmm15",
+ "mov ecx, 32",
+ "20:",
+ "vmovdqu ymm0, YMMWORD PTR [rdi]",
+ "vmovdqu ymm1, YMMWORD PTR [rsi]",
+ "vpsubd ymm0, ymm0, ymm1",
+ "vpsrad ymm2, ymm0, 31",
+ "vpand ymm2, ymm2, ymm15",
+ "vpaddd ymm0, ymm0, ymm2",
+ "vmovdqu YMMWORD PTR [rdi], ymm0",
+ "add rdi, 32",
+ "add rsi, 32",
+ "sub rcx, 1",
+ "jne 20b",
+ "vzeroupper",
+ "ret",
+ )
+}
+
/// `SimpleBitPack(f, b)` (FIPS 204 Algorithm 16): writes the 256 coefficients of `*f` to `out` as `bitlen b`-bit little-endian fields.
///
/// Contract: `VG.Spec.MlDsa.simpleBitPackContract`. Constant time: only the pointers, `b` and `len` may affect timing, not the data.
diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs
index 63ea03f6e..90e8d05f4 100644
--- a/src/asm/x86_64/mldsa44.rs
+++ b/src/asm/x86_64/mldsa44.rs
@@ -2,6 +2,3460 @@
//! Verified `mldsa44` functions for `x86_64`.
#![allow(dead_code)]
+/// The CPU features `vg_mldsa44_keygen_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA44_KEYGEN_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-44 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
+///
+/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys.
+///
+/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address.
+///
+/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails.
+///
+/// # Safety
+///
+/// * `seed` must be valid for reads of 32 bytes.
+/// * `pk` must be valid for reads and writes of 1312 bytes.
+/// * `sk` must be valid for reads and writes of 2560 bytes.
+/// * `scratch` must be valid for reads and writes of 77824 bytes.
+/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do).
+/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32], pk: *mut [u8; 1312], sk: *mut [u8; 2560], scratch: *mut [u64; 9728]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [rcx+840], rbx",
+ "mov QWORD PTR [rcx+848], rbp",
+ "mov QWORD PTR [rcx+856], r12",
+ "mov QWORD PTR [rcx+864], r13",
+ "mov QWORD PTR [rcx+872], r14",
+ "mov QWORD PTR [rcx+880], r15",
+ "mov rbx, rcx",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r15d, 1",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+896], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+897], al",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbp",
+ "add rcx, 0",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 32",
+ "mov rcx, rbx",
+ "add rcx, 896",
+ "mov r8d, 2",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 34",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 1024",
+ "mov r8d, 128",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "20:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 20b",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov rsi, rbx",
+ "add rsi, 1056",
+ "mov ecx, 64",
+ "21:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 21b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1281], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 4096",
+ "mov ecx, 256",
+ "22:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 22b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 5120",
+ "mov ecx, 256",
+ "23:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 23b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov ecx, 256",
+ "24:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 24b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 7168",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov ecx, 256",
+ "25:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov ecx, 256",
+ "26:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 26b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 9216",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov ecx, 256",
+ "27:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 27b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 10240",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 10240",
+ "mov ecx, 256",
+ "28:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 28b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 11264",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 11264",
+ "mov ecx, 256",
+ "29:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 29b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 12288",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 12288",
+ "mov ecx, 256",
+ "210:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 210b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 13312",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 13312",
+ "mov ecx, 256",
+ "211:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 211b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 14336",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 14336",
+ "mov ecx, 256",
+ "212:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 212b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 15360",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 15360",
+ "mov ecx, 256",
+ "213:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 213b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 16384",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov ecx, 256",
+ "214:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 214b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 17408",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov ecx, 256",
+ "215:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 215b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 18432",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov ecx, 256",
+ "216:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 216b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 19456",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov ecx, 256",
+ "217:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 217b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov ecx, 256",
+ "218:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 218b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov ecx, 256",
+ "219:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 219b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov ecx, 256",
+ "220:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 220b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov ecx, 256",
+ "221:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 221b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov ecx, 256",
+ "222:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 222b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov ecx, 256",
+ "223:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 223b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov ecx, 256",
+ "224:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 224b",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov ecx, 256",
+ "225:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 225b",
+ "mov rdi, r12",
+ "add rdi, 0",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "226:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 226b",
+ "mov rdi, r13",
+ "add rdi, 0",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "227:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 227b",
+ "mov rdi, r13",
+ "add rdi, 32",
+ "mov rsi, rbx",
+ "add rsi, 1120",
+ "mov ecx, 32",
+ "228:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 228b",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 128",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 224",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 320",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 416",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 512",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 608",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 704",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 800",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 7168",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 32",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 896",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 9216",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 10240",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 11264",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 352",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 1312",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 12288",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 13312",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 14336",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 15360",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 672",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 1728",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 16384",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 17408",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 18432",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 19456",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 27648",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 992",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 2144",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 1312",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 88",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r13",
+ "add rcx, 64",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
+ vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2,
+ vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ )
+}
+
+/// The CPU features `vg_mldsa44_sign_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA44_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-44 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
+///
+/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness.
+///
+/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address.
+///
+/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing.
+///
+/// # Safety
+///
+/// * `sk` must be valid for reads of 2560 bytes.
+/// * `mu` must be valid for reads of 64 bytes.
+/// * `rnd` must be valid for reads of 32 bytes.
+/// * `sig` must be valid for reads and writes of 2420 bytes.
+/// * `scratch` must be valid for reads and writes of 77824 bytes.
+/// * `sk` must have been written by `vg_mldsa44_keygen`.
+/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do).
+/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 2420], scratch: *mut [u64; 9728]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [r8+840], rbx",
+ "mov QWORD PTR [r8+848], rbp",
+ "mov QWORD PTR [r8+856], r12",
+ "mov QWORD PTR [r8+864], r13",
+ "mov QWORD PTR [r8+872], r14",
+ "mov QWORD PTR [r8+880], r15",
+ "mov rbx, r8",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r14, rcx",
+ "mov r15d, 1",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbp",
+ "add rsi, 0",
+ "mov ecx, 32",
+ "20:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 20b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 41984",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 43008",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 44032",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 49152",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 50176",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 51200",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 52224",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "test r15d, r15d",
+ "jne 21f",
+ "jmp 22f",
+ "21:",
+ "mov rdi, rbp",
+ "add rdi, 128",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 26624",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 224",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 27648",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 320",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 28672",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 416",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 29696",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 512",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 30720",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 608",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 31744",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 704",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 32768",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 800",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 33792",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 896",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 34816",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1312",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 35840",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1728",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 36864",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 2144",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 37888",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbp",
+ "add rcx, 32",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 32",
+ "mov rcx, r13",
+ "add rcx, 0",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 128",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 960",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx+896], rax",
+ "mov eax, 814",
+ "mov QWORD PTR [rbx+888], rax",
+ "23:",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 0",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 131072",
+ "mov rdx, rbx",
+ "add rdx, 14336",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov rsi, rbx",
+ "add rsi, 14336",
+ "mov ecx, 1024",
+ "24:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 24b",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 1",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 131072",
+ "mov rdx, rbx",
+ "add rdx, 15360",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov rsi, rbx",
+ "add rsi, 15360",
+ "mov ecx, 1024",
+ "25:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 2",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 131072",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov rsi, rbx",
+ "add rsi, 16384",
+ "mov ecx, 1024",
+ "26:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 26b",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 3",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 131072",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov rsi, rbx",
+ "add rsi, 17408",
+ "mov ecx, 1024",
+ "27:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 27b",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 41984",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 43008",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 44032",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 49152",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 50176",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 51200",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 52224",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov esi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 43",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "mov ecx, 192",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov esi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 43",
+ "mov rdx, rbx",
+ "add rdx, 2240",
+ "mov ecx, 192",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov esi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 43",
+ "mov rdx, rbx",
+ "add rdx, 2432",
+ "mov ecx, 192",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov esi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 43",
+ "mov rdx, rbx",
+ "add rdx, 2624",
+ "mov ecx, 192",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "mov r8d, 768",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 16",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 1040",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov rdi, rbx",
+ "add rdi, 1040",
+ "mov esi, 32",
+ "mov edx, 39",
+ "mov rcx, rbx",
+ "add rcx, 5120",
+ "mov r8, rbx",
+ "add r8, 3072",
+ "call {vg_mldsa_sample_in_ball}",
+ "test eax, eax",
+ "jne 28f",
+ "mov r15d, 0",
+ "mov eax, 1",
+ "mov QWORD PTR [rbx+888], rax",
+ "jmp 29f",
+ "28:",
+ "mov rdi, rbx",
+ "add rdi, 5120",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov r15d, 1",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx+904], rax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 14336",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 14336",
+ "mov esi, 130994",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 15360",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 15360",
+ "mov esi, 130994",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov esi, 130994",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov esi, 130994",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov esi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 95154",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov esi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 95154",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 32768",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov esi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 95154",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 33792",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov esi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 95154",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 34816",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 95232",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 22528",
+ "mov ecx, 1024",
+ "210:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 210b",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 22528",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 22528",
+ "mov edx, 95232",
+ "mov rcx, rbx",
+ "add rcx, 10240",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 35840",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 95232",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov ecx, 1024",
+ "211:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 211b",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov edx, 95232",
+ "mov rcx, rbx",
+ "add rcx, 11264",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 36864",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 95232",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "mov ecx, 1024",
+ "212:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 212b",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "mov edx, 95232",
+ "mov rcx, rbx",
+ "add rcx, 12288",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 37888",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 95232",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "mov ecx, 1024",
+ "213:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 213b",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "mov edx, 95232",
+ "mov rcx, rbx",
+ "add rcx, 13312",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov eax, DWORD PTR [rbx+904]",
+ "sub rax, 81",
+ "shr rax, 63",
+ "and r15d, eax",
+ "test r15d, r15d",
+ "jne 214f",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 4",
+ "mov QWORD PTR [rbx+896], rax",
+ "jmp 215f",
+ "214:",
+ "mov eax, 1",
+ "mov QWORD PTR [rbx+888], rax",
+ "215:",
+ "29:",
+ "mov rax, QWORD PTR [rbx+888]",
+ "sub rax, 1",
+ "mov QWORD PTR [rbx+888], rax",
+ "jne 23b",
+ "test r15d, r15d",
+ "jne 216f",
+ "jmp 217f",
+ "216:",
+ "mov rdi, r14",
+ "add rdi, 0",
+ "mov rsi, rbx",
+ "add rsi, 1040",
+ "mov ecx, 32",
+ "218:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 218b",
+ "mov rdi, rbx",
+ "add rdi, 14336",
+ "mov esi, 131071",
+ "mov edx, 131072",
+ "mov rcx, r14",
+ "add rcx, 32",
+ "mov r8d, 576",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 15360",
+ "mov esi, 131071",
+ "mov edx, 131072",
+ "mov rcx, r14",
+ "add rcx, 608",
+ "mov r8d, 576",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov esi, 131071",
+ "mov edx, 131072",
+ "mov rcx, r14",
+ "add rcx, 1184",
+ "mov r8d, 576",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov esi, 131071",
+ "mov edx, 131072",
+ "mov rcx, r14",
+ "add rcx, 1760",
+ "mov r8d, 576",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 10240",
+ "mov esi, 1024",
+ "mov edx, 80",
+ "mov rcx, r14",
+ "add rcx, 2336",
+ "mov r8d, 84",
+ "call {vg_mldsa_hint_bit_pack}",
+ "217:",
+ "22:",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_high_bits = sym super::mldsa::vg_mldsa_high_bits,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball,
+ vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2,
+ vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt,
+ vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2,
+ vg_mldsa_low_bits = sym super::mldsa::vg_mldsa_low_bits,
+ vg_mldsa_make_hint = sym super::mldsa::vg_mldsa_make_hint,
+ vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
+ vg_mldsa_hint_bit_pack = sym super::mldsa::vg_mldsa_hint_bit_pack,
+ )
+}
+
+/// The CPU features `vg_mldsa44_verify_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA44_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-44 verification of a signature of a message representative, `ML-DSA.Verify_internal(pk, M′, σ)` (FIPS 204 Algorithm 8) with `μ` computed by the caller: with the public key `*pk`, the 64-byte message representative `μ = H(H(pk, 64) ‖ M′, 64)` at `mu` and the signature `*sig`, returns 1 if the signature is valid, and 0 if it is not or if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less).
+///
+/// Contract: `VG.Spec.MlDsa.verifyContract`. Not constant time: timing may depend on the public key, the message representative and the signature.
+///
+/// It calls the `vg_mldsa_*` primitives and the SHAKE256 sponge. The samplers' results are combined without a branch, so the only branches depend on the public key and the signature.
+///
+/// # Safety
+///
+/// * `pk` must be valid for reads of 1312 bytes.
+/// * `mu` must be valid for reads of 64 bytes.
+/// * `sig` must be valid for reads of 2420 bytes.
+/// * `scratch` must be valid for reads and writes of 77824 bytes.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do).
+/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312], mu: *const [u8; 64], sig: *const [u8; 2420], scratch: *mut [u64; 9728]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [rcx+840], rbx",
+ "mov QWORD PTR [rcx+848], rbp",
+ "mov QWORD PTR [rcx+856], r12",
+ "mov QWORD PTR [rcx+864], r13",
+ "mov QWORD PTR [rcx+872], r14",
+ "mov QWORD PTR [rcx+880], r15",
+ "mov rbx, rcx",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r15d, 1",
+ "mov rdi, r13",
+ "add rdi, 2336",
+ "mov esi, 84",
+ "mov edx, 80",
+ "mov rcx, rbx",
+ "add rcx, 8192",
+ "mov r8d, 1024",
+ "call {vg_mldsa_hint_bit_unpack}",
+ "mov r15d, eax",
+ "test r15d, r15d",
+ "jne 20f",
+ "jmp 21f",
+ "20:",
+ "mov rdi, r13",
+ "add rdi, 32",
+ "mov esi, 576",
+ "mov edx, 131071",
+ "mov ecx, 131072",
+ "mov r8, rbx",
+ "add r8, 16384",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov esi, 130994",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 608",
+ "mov esi, 576",
+ "mov edx, 131071",
+ "mov ecx, 131072",
+ "mov r8, rbx",
+ "add r8, 17408",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov esi, 130994",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 1184",
+ "mov esi, 576",
+ "mov edx, 131071",
+ "mov ecx, 131072",
+ "mov r8, rbx",
+ "add r8, 18432",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov esi, 130994",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 1760",
+ "mov esi, 576",
+ "mov edx, 131071",
+ "mov ecx, 131072",
+ "mov r8, rbx",
+ "add r8, 19456",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov esi, 130994",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "test r15d, r15d",
+ "jne 22f",
+ "jmp 23f",
+ "22:",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbp",
+ "add rsi, 0",
+ "mov ecx, 32",
+ "24:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 24b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov ecx, 256",
+ "25:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov ecx, 256",
+ "26:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 26b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov ecx, 256",
+ "27:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 27b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov ecx, 256",
+ "28:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 28b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov ecx, 256",
+ "29:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 29b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov ecx, 256",
+ "210:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 210b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov ecx, 256",
+ "211:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 211b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov ecx, 256",
+ "212:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 212b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 45056",
+ "mov ecx, 256",
+ "213:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 213b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov ecx, 256",
+ "214:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 214b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov ecx, 256",
+ "215:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 215b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov ecx, 256",
+ "216:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 216b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 53248",
+ "mov ecx, 256",
+ "217:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 217b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 54272",
+ "mov ecx, 256",
+ "218:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 218b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 55296",
+ "mov ecx, 256",
+ "219:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 219b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 56320",
+ "mov ecx, 256",
+ "220:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 220b",
+ "mov rdi, r13",
+ "add rdi, 0",
+ "mov esi, 32",
+ "mov edx, 39",
+ "mov rcx, rbx",
+ "add rcx, 23552",
+ "mov r8, rbx",
+ "add r8, 4096",
+ "call {vg_mldsa_sample_in_ball}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov ecx, 256",
+ "221:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 221b",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 32",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 95232",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 43",
+ "mov rdx, rbx",
+ "add rdx, 1024",
+ "mov ecx, 192",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 352",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 95232",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 43",
+ "mov rdx, rbx",
+ "add rdx, 1216",
+ "mov ecx, 192",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 672",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 10240",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 95232",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 43",
+ "mov rdx, rbx",
+ "add rdx, 1408",
+ "mov ecx, 192",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 992",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 11264",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 95232",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 43",
+ "mov rdx, rbx",
+ "add rdx, 1600",
+ "mov ecx, 192",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, rbx",
+ "add rcx, 1024",
+ "mov r8d, 768",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 16",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "mov rdi, r13",
+ "add rdi, 0",
+ "mov ecx, 32",
+ "mov edx, 0",
+ "222:",
+ "movzx eax, BYTE PTR [rsi]",
+ "movzx r8d, BYTE PTR [rdi]",
+ "xor rax, r8",
+ "or rdx, rax",
+ "add rsi, 1",
+ "add rdi, 1",
+ "sub rcx, 1",
+ "jne 222b",
+ "sub rdx, 1",
+ "sbb rax, rax",
+ "and r15d, eax",
+ "23:",
+ "21:",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack,
+ vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack,
+ vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt,
+ vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1,
+ vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ )
+}
+
/// ML-DSA-44 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
///
/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys.
diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs
index 6ff94ded7..6452825aa 100644
--- a/src/asm/x86_64/mldsa65.rs
+++ b/src/asm/x86_64/mldsa65.rs
@@ -2,6 +2,5175 @@
//! Verified `mldsa65` functions for `x86_64`.
#![allow(dead_code)]
+/// The CPU features `vg_mldsa65_keygen_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA65_KEYGEN_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-65 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
+///
+/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys.
+///
+/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address.
+///
+/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails.
+///
+/// # Safety
+///
+/// * `seed` must be valid for reads of 32 bytes.
+/// * `pk` must be valid for reads and writes of 1952 bytes.
+/// * `sk` must be valid for reads and writes of 4032 bytes.
+/// * `scratch` must be valid for reads and writes of 103424 bytes.
+/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do).
+/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32], pk: *mut [u8; 1952], sk: *mut [u8; 4032], scratch: *mut [u64; 12928]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [rcx+840], rbx",
+ "mov QWORD PTR [rcx+848], rbp",
+ "mov QWORD PTR [rcx+856], r12",
+ "mov QWORD PTR [rcx+864], r13",
+ "mov QWORD PTR [rcx+872], r14",
+ "mov QWORD PTR [rcx+880], r15",
+ "mov rbx, rcx",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r15d, 1",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+896], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+897], al",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbp",
+ "add rcx, 0",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 32",
+ "mov rcx, rbx",
+ "add rcx, 896",
+ "mov r8d, 2",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 34",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 1024",
+ "mov r8d, 128",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "20:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 20b",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov rsi, rbx",
+ "add rsi, 1056",
+ "mov ecx, 64",
+ "21:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 21b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1281], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 4096",
+ "mov ecx, 256",
+ "22:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 22b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 5120",
+ "mov ecx, 256",
+ "23:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 23b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov ecx, 256",
+ "24:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 24b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 7168",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov ecx, 256",
+ "25:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov ecx, 256",
+ "26:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 26b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 9216",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov ecx, 256",
+ "27:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 27b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 10240",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 10240",
+ "mov ecx, 256",
+ "28:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 28b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 11264",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 11264",
+ "mov ecx, 256",
+ "29:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 29b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 12288",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 12288",
+ "mov ecx, 256",
+ "210:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 210b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 13312",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 13312",
+ "mov ecx, 256",
+ "211:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 211b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 14336",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 14336",
+ "mov ecx, 256",
+ "212:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 212b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 15360",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 15360",
+ "mov ecx, 256",
+ "213:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 213b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 16384",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov ecx, 256",
+ "214:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 214b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 17408",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov ecx, 256",
+ "215:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 215b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 18432",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov ecx, 256",
+ "216:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 216b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 19456",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov ecx, 256",
+ "217:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 217b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 20480",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov ecx, 256",
+ "218:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 218b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 21504",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov ecx, 256",
+ "219:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 219b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 22528",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov ecx, 256",
+ "220:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 220b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov ecx, 256",
+ "221:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 221b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov ecx, 256",
+ "222:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 222b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov ecx, 256",
+ "223:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 223b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov ecx, 256",
+ "224:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 224b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 27648",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov ecx, 256",
+ "225:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 225b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov ecx, 256",
+ "226:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 226b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov ecx, 256",
+ "227:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 227b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov ecx, 256",
+ "228:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 228b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov ecx, 256",
+ "229:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 229b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov ecx, 256",
+ "230:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 230b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov ecx, 256",
+ "231:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 231b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 34816",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov ecx, 256",
+ "232:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 232b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 35840",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov ecx, 256",
+ "233:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 233b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 36864",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov ecx, 256",
+ "234:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 234b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 37888",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov ecx, 256",
+ "235:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 235b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 38912",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov ecx, 256",
+ "236:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 236b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 39936",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov ecx, 256",
+ "237:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 237b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 40960",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 40960",
+ "mov ecx, 256",
+ "238:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 238b",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 41984",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 41984",
+ "mov ecx, 256",
+ "239:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 239b",
+ "mov eax, 8",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 43008",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 43008",
+ "mov ecx, 256",
+ "240:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 240b",
+ "mov eax, 9",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 44032",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 44032",
+ "mov ecx, 256",
+ "241:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 241b",
+ "mov eax, 10",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 4",
+ "mov rdx, rbx",
+ "add rdx, 45056",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 45056",
+ "mov ecx, 256",
+ "242:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 242b",
+ "mov rdi, r12",
+ "add rdi, 0",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "243:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 243b",
+ "mov rdi, r13",
+ "add rdi, 0",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "244:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 244b",
+ "mov rdi, r13",
+ "add rdi, 32",
+ "mov rsi, rbx",
+ "add rsi, 1120",
+ "mov ecx, 32",
+ "245:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 245b",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 128",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 256",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 384",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 512",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 640",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 768",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 40960",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 896",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 41984",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 1024",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 43008",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 1152",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 44032",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 1280",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 45056",
+ "mov esi, 4",
+ "mov edx, 4",
+ "mov rcx, r13",
+ "add rcx, 1408",
+ "mov r8d, 128",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "mov rdx, rbx",
+ "add rdx, 34816",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 35840",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "mov rdx, rbx",
+ "add rdx, 36864",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 7168",
+ "mov rdx, rbx",
+ "add rdx, 37888",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "mov rdx, rbx",
+ "add rdx, 38912",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 48128",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 32",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 1536",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 9216",
+ "mov rdx, rbx",
+ "add rdx, 34816",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 10240",
+ "mov rdx, rbx",
+ "add rdx, 35840",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 11264",
+ "mov rdx, rbx",
+ "add rdx, 36864",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 12288",
+ "mov rdx, rbx",
+ "add rdx, 37888",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 13312",
+ "mov rdx, rbx",
+ "add rdx, 38912",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 48128",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 352",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 1952",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 14336",
+ "mov rdx, rbx",
+ "add rdx, 34816",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 15360",
+ "mov rdx, rbx",
+ "add rdx, 35840",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 16384",
+ "mov rdx, rbx",
+ "add rdx, 36864",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 17408",
+ "mov rdx, rbx",
+ "add rdx, 37888",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 18432",
+ "mov rdx, rbx",
+ "add rdx, 38912",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 41984",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 48128",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 672",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 2368",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 19456",
+ "mov rdx, rbx",
+ "add rdx, 34816",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 20480",
+ "mov rdx, rbx",
+ "add rdx, 35840",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 21504",
+ "mov rdx, rbx",
+ "add rdx, 36864",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 22528",
+ "mov rdx, rbx",
+ "add rdx, 37888",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 38912",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 43008",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 48128",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 992",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 2784",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "mov rdx, rbx",
+ "add rdx, 34816",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "mov rdx, rbx",
+ "add rdx, 35840",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov rdx, rbx",
+ "add rdx, 36864",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 27648",
+ "mov rdx, rbx",
+ "add rdx, 37888",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 38912",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 44032",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 48128",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 1312",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 3200",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 34816",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 35840",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 36864",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov rdx, rbx",
+ "add rdx, 37888",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "mov rdx, rbx",
+ "add rdx, 38912",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 48128",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 1632",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 3616",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 1952",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 48",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r13",
+ "add rcx, 64",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
+ vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2,
+ vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ )
+}
+
+/// The CPU features `vg_mldsa65_sign_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA65_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-65 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
+///
+/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness.
+///
+/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address.
+///
+/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing.
+///
+/// # Safety
+///
+/// * `sk` must be valid for reads of 4032 bytes.
+/// * `mu` must be valid for reads of 64 bytes.
+/// * `rnd` must be valid for reads of 32 bytes.
+/// * `sig` must be valid for reads and writes of 3309 bytes.
+/// * `scratch` must be valid for reads and writes of 103424 bytes.
+/// * `sk` must have been written by `vg_mldsa65_keygen`.
+/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do).
+/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 3309], scratch: *mut [u64; 12928]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [r8+840], rbx",
+ "mov QWORD PTR [r8+848], rbp",
+ "mov QWORD PTR [r8+856], r12",
+ "mov QWORD PTR [r8+864], r13",
+ "mov QWORD PTR [r8+872], r14",
+ "mov QWORD PTR [r8+880], r15",
+ "mov rbx, r8",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r14, rcx",
+ "mov r15d, 1",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbp",
+ "add rsi, 0",
+ "mov ecx, 32",
+ "20:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 20b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 50176",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 51200",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 52224",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 57344",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 58368",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 59392",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 60416",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 61440",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 62464",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 63488",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 64512",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 65536",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 66560",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 67584",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 68608",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 69632",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 73728",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 74752",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 75776",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 76800",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 78848",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 79872",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "test r15d, r15d",
+ "jne 21f",
+ "jmp 22f",
+ "21:",
+ "mov rdi, rbp",
+ "add rdi, 128",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 32768",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 256",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 33792",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 384",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 34816",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 512",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 35840",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 640",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 36864",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 768",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 37888",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 896",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 38912",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1024",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 39936",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1152",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 40960",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 40960",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1280",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 41984",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 41984",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1408",
+ "mov esi, 128",
+ "mov edx, 4",
+ "mov ecx, 4",
+ "mov r8, rbx",
+ "add r8, 43008",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 43008",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1536",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 44032",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 44032",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1952",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 45056",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 45056",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 2368",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 46080",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 2784",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 47104",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 3200",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 48128",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 3616",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 49152",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 49152",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbp",
+ "add rcx, 32",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 32",
+ "mov rcx, r13",
+ "add rcx, 0",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 128",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 960",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx+896], rax",
+ "mov eax, 814",
+ "mov QWORD PTR [rbx+888], rax",
+ "23:",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 0",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov rsi, rbx",
+ "add rsi, 16384",
+ "mov ecx, 1024",
+ "24:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 24b",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 1",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 17408",
+ "mov ecx, 1024",
+ "25:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 2",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 18432",
+ "mov ecx, 1024",
+ "26:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 26b",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 3",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 19456",
+ "mov ecx, 1024",
+ "27:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 27b",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 4",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 20480",
+ "mov ecx, 1024",
+ "28:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 28b",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 50176",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 51200",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 52224",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 57344",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 58368",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 59392",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 60416",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 61440",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 62464",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 63488",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 64512",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 65536",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 66560",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 67584",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 68608",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 69632",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 73728",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 74752",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 75776",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 76800",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 78848",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 79872",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2176",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2304",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2432",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2560",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2688",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "mov r8d, 768",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 16",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 1040",
+ "mov r8d, 48",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov rdi, rbx",
+ "add rdi, 1040",
+ "mov esi, 48",
+ "mov edx, 49",
+ "mov rcx, rbx",
+ "add rcx, 5120",
+ "mov r8, rbx",
+ "add r8, 3072",
+ "call {vg_mldsa_sample_in_ball}",
+ "test eax, eax",
+ "jne 29f",
+ "mov r15d, 0",
+ "mov eax, 1",
+ "mov QWORD PTR [rbx+888], rax",
+ "jmp 210f",
+ "29:",
+ "mov rdi, rbx",
+ "add rdi, 5120",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov r15d, 1",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx+904], rax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 32768",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 33792",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 34816",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 35840",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 36864",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 37888",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261692",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 38912",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261692",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 39936",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261692",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 40960",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261692",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 41984",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261692",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 43008",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261692",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 44032",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov ecx, 1024",
+ "211:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 211b",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 10240",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 45056",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 27648",
+ "mov ecx, 1024",
+ "212:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 212b",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 27648",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 27648",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 11264",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 46080",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov ecx, 1024",
+ "213:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 213b",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 12288",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 47104",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov ecx, 1024",
+ "214:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 214b",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 13312",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 48128",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov ecx, 1024",
+ "215:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 215b",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 14336",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 49152",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov ecx, 1024",
+ "216:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 216b",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 15360",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov eax, DWORD PTR [rbx+904]",
+ "sub rax, 56",
+ "shr rax, 63",
+ "and r15d, eax",
+ "test r15d, r15d",
+ "jne 217f",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 5",
+ "mov QWORD PTR [rbx+896], rax",
+ "jmp 218f",
+ "217:",
+ "mov eax, 1",
+ "mov QWORD PTR [rbx+888], rax",
+ "218:",
+ "210:",
+ "mov rax, QWORD PTR [rbx+888]",
+ "sub rax, 1",
+ "mov QWORD PTR [rbx+888], rax",
+ "jne 23b",
+ "test r15d, r15d",
+ "jne 219f",
+ "jmp 220f",
+ "219:",
+ "mov rdi, r14",
+ "add rdi, 0",
+ "mov rsi, rbx",
+ "add rsi, 1040",
+ "mov ecx, 48",
+ "221:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 221b",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 48",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 688",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 1328",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 1968",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 2608",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 10240",
+ "mov esi, 1536",
+ "mov edx, 55",
+ "mov rcx, r14",
+ "add rcx, 3248",
+ "mov r8d, 61",
+ "call {vg_mldsa_hint_bit_pack}",
+ "220:",
+ "22:",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_high_bits = sym super::mldsa::vg_mldsa_high_bits,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball,
+ vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2,
+ vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt,
+ vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2,
+ vg_mldsa_low_bits = sym super::mldsa::vg_mldsa_low_bits,
+ vg_mldsa_make_hint = sym super::mldsa::vg_mldsa_make_hint,
+ vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
+ vg_mldsa_hint_bit_pack = sym super::mldsa::vg_mldsa_hint_bit_pack,
+ )
+}
+
+/// The CPU features `vg_mldsa65_verify_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA65_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-65 verification of a signature of a message representative, `ML-DSA.Verify_internal(pk, M′, σ)` (FIPS 204 Algorithm 8) with `μ` computed by the caller: with the public key `*pk`, the 64-byte message representative `μ = H(H(pk, 64) ‖ M′, 64)` at `mu` and the signature `*sig`, returns 1 if the signature is valid, and 0 if it is not or if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less).
+///
+/// Contract: `VG.Spec.MlDsa.verifyContract`. Not constant time: timing may depend on the public key, the message representative and the signature.
+///
+/// It calls the `vg_mldsa_*` primitives and the SHAKE256 sponge. The samplers' results are combined without a branch, so the only branches depend on the public key and the signature.
+///
+/// # Safety
+///
+/// * `pk` must be valid for reads of 1952 bytes.
+/// * `mu` must be valid for reads of 64 bytes.
+/// * `sig` must be valid for reads of 3309 bytes.
+/// * `scratch` must be valid for reads and writes of 103424 bytes.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do).
+/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952], mu: *const [u8; 64], sig: *const [u8; 3309], scratch: *mut [u64; 12928]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [rcx+840], rbx",
+ "mov QWORD PTR [rcx+848], rbp",
+ "mov QWORD PTR [rcx+856], r12",
+ "mov QWORD PTR [rcx+864], r13",
+ "mov QWORD PTR [rcx+872], r14",
+ "mov QWORD PTR [rcx+880], r15",
+ "mov rbx, rcx",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r15d, 1",
+ "mov rdi, r13",
+ "add rdi, 3248",
+ "mov esi, 61",
+ "mov edx, 55",
+ "mov rcx, rbx",
+ "add rcx, 8192",
+ "mov r8d, 1536",
+ "call {vg_mldsa_hint_bit_unpack}",
+ "mov r15d, eax",
+ "test r15d, r15d",
+ "jne 20f",
+ "jmp 21f",
+ "20:",
+ "mov rdi, r13",
+ "add rdi, 48",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 16384",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 688",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 17408",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 1328",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 18432",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 1968",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 19456",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 2608",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 20480",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov esi, 524092",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "test r15d, r15d",
+ "jne 22f",
+ "jmp 23f",
+ "22:",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbp",
+ "add rsi, 0",
+ "mov ecx, 32",
+ "24:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 24b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov ecx, 256",
+ "25:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov ecx, 256",
+ "26:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 26b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov ecx, 256",
+ "27:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 27b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov ecx, 256",
+ "28:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 28b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov ecx, 256",
+ "29:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 29b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov ecx, 256",
+ "210:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 210b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov ecx, 256",
+ "211:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 211b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov ecx, 256",
+ "212:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 212b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov ecx, 256",
+ "213:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 213b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 40960",
+ "mov ecx, 256",
+ "214:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 214b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 45056",
+ "mov ecx, 256",
+ "215:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 215b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov ecx, 256",
+ "216:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 216b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov ecx, 256",
+ "217:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 217b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov ecx, 256",
+ "218:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 218b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 49152",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 49152",
+ "mov ecx, 256",
+ "219:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 219b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 53248",
+ "mov ecx, 256",
+ "220:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 220b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 54272",
+ "mov ecx, 256",
+ "221:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 221b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 55296",
+ "mov ecx, 256",
+ "222:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 222b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 56320",
+ "mov ecx, 256",
+ "223:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 223b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 57344",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 57344",
+ "mov ecx, 256",
+ "224:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 224b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 61440",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 61440",
+ "mov ecx, 256",
+ "225:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 225b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 62464",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 62464",
+ "mov ecx, 256",
+ "226:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 226b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 63488",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 63488",
+ "mov ecx, 256",
+ "227:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 227b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 64512",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 64512",
+ "mov ecx, 256",
+ "228:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 228b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 65536",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 65536",
+ "mov ecx, 256",
+ "229:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 229b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 69632",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 69632",
+ "mov ecx, 256",
+ "230:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 230b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 70656",
+ "mov ecx, 256",
+ "231:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 231b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 71680",
+ "mov ecx, 256",
+ "232:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 232b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 72704",
+ "mov ecx, 256",
+ "233:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 233b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 73728",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 73728",
+ "mov ecx, 256",
+ "234:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 234b",
+ "mov rdi, r13",
+ "add rdi, 0",
+ "mov esi, 48",
+ "mov edx, 49",
+ "mov rcx, rbx",
+ "add rcx, 23552",
+ "mov r8, rbx",
+ "add r8, 4096",
+ "call {vg_mldsa_sample_in_ball}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov ecx, 256",
+ "235:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 235b",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 32",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1024",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 352",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1152",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 49152",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 672",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 10240",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1280",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 57344",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 992",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 11264",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1408",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 61440",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 62464",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 63488",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 64512",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 65536",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1312",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 12288",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1536",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 69632",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 73728",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1632",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 13312",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1664",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, rbx",
+ "add rcx, 1024",
+ "mov r8d, 768",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 16",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "mov r8d, 48",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "mov rdi, r13",
+ "add rdi, 0",
+ "mov ecx, 48",
+ "mov edx, 0",
+ "236:",
+ "movzx eax, BYTE PTR [rsi]",
+ "movzx r8d, BYTE PTR [rdi]",
+ "xor rax, r8",
+ "or rdx, rax",
+ "add rsi, 1",
+ "add rdi, 1",
+ "sub rcx, 1",
+ "jne 236b",
+ "sub rdx, 1",
+ "sbb rax, rax",
+ "and r15d, eax",
+ "23:",
+ "21:",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack,
+ vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack,
+ vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt,
+ vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1,
+ vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ )
+}
+
/// ML-DSA-65 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
///
/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys.
diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs
index be3306d5c..79f684dd0 100644
--- a/src/asm/x86_64/mldsa87.rs
+++ b/src/asm/x86_64/mldsa87.rs
@@ -2,6 +2,7989 @@
//! Verified `mldsa87` functions for `x86_64`.
#![allow(dead_code)]
+/// The CPU features `vg_mldsa87_keygen_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA87_KEYGEN_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
+///
+/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys.
+///
+/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address.
+///
+/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails.
+///
+/// # Safety
+///
+/// * `seed` must be valid for reads of 32 bytes.
+/// * `pk` must be valid for reads and writes of 2592 bytes.
+/// * `sk` must be valid for reads and writes of 4896 bytes.
+/// * `scratch` must be valid for reads and writes of 144384 bytes.
+/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do).
+/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32], pk: *mut [u8; 2592], sk: *mut [u8; 4896], scratch: *mut [u64; 18048]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [rcx+840], rbx",
+ "mov QWORD PTR [rcx+848], rbp",
+ "mov QWORD PTR [rcx+856], r12",
+ "mov QWORD PTR [rcx+864], r13",
+ "mov QWORD PTR [rcx+872], r14",
+ "mov QWORD PTR [rcx+880], r15",
+ "mov rbx, rcx",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r15d, 1",
+ "mov eax, 8",
+ "mov BYTE PTR [rbx+896], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+897], al",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbp",
+ "add rcx, 0",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 32",
+ "mov rcx, rbx",
+ "add rcx, 896",
+ "mov r8d, 2",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 34",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 1024",
+ "mov r8d, 128",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "20:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 20b",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov rsi, rbx",
+ "add rsi, 1056",
+ "mov ecx, 64",
+ "21:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 21b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1281], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 4096",
+ "mov ecx, 256",
+ "22:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 22b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 5120",
+ "mov ecx, 256",
+ "23:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 23b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov ecx, 256",
+ "24:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 24b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 7168",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov ecx, 256",
+ "25:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov ecx, 256",
+ "26:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 26b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 9216",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov ecx, 256",
+ "27:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 27b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 10240",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 10240",
+ "mov ecx, 256",
+ "28:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 28b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 11264",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 11264",
+ "mov ecx, 256",
+ "29:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 29b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 12288",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 12288",
+ "mov ecx, 256",
+ "210:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 210b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 13312",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 13312",
+ "mov ecx, 256",
+ "211:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 211b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 14336",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 14336",
+ "mov ecx, 256",
+ "212:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 212b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 15360",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 15360",
+ "mov ecx, 256",
+ "213:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 213b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 16384",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov ecx, 256",
+ "214:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 214b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 17408",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov ecx, 256",
+ "215:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 215b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 18432",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov ecx, 256",
+ "216:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 216b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 19456",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov ecx, 256",
+ "217:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 217b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 20480",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov ecx, 256",
+ "218:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 218b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 21504",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov ecx, 256",
+ "219:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 219b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 22528",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov ecx, 256",
+ "220:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 220b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov ecx, 256",
+ "221:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 221b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov ecx, 256",
+ "222:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 222b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov ecx, 256",
+ "223:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 223b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov ecx, 256",
+ "224:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 224b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 27648",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov ecx, 256",
+ "225:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 225b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov ecx, 256",
+ "226:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 226b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov ecx, 256",
+ "227:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 227b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov ecx, 256",
+ "228:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 228b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov ecx, 256",
+ "229:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 229b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov ecx, 256",
+ "230:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 230b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov ecx, 256",
+ "231:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 231b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 34816",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov ecx, 256",
+ "232:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 232b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 35840",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov ecx, 256",
+ "233:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 233b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov ecx, 256",
+ "234:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 234b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov ecx, 256",
+ "235:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 235b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov ecx, 256",
+ "236:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 236b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov ecx, 256",
+ "237:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 237b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 40960",
+ "mov ecx, 256",
+ "238:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 238b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 41984",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 41984",
+ "mov ecx, 256",
+ "239:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 239b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 43008",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 43008",
+ "mov ecx, 256",
+ "240:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 240b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 44032",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 44032",
+ "mov ecx, 256",
+ "241:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 241b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 45056",
+ "mov ecx, 256",
+ "242:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 242b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov ecx, 256",
+ "243:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 243b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov ecx, 256",
+ "244:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 244b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov ecx, 256",
+ "245:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 245b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 49152",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 49152",
+ "mov ecx, 256",
+ "246:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 246b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 50176",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 50176",
+ "mov ecx, 256",
+ "247:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 247b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 51200",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 51200",
+ "mov ecx, 256",
+ "248:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 248b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 52224",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 52224",
+ "mov ecx, 256",
+ "249:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 249b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 53248",
+ "mov ecx, 256",
+ "250:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 250b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 54272",
+ "mov ecx, 256",
+ "251:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 251b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 55296",
+ "mov ecx, 256",
+ "252:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 252b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 56320",
+ "mov ecx, 256",
+ "253:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 253b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 57344",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 57344",
+ "mov ecx, 256",
+ "254:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 254b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 58368",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 58368",
+ "mov ecx, 256",
+ "255:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 255b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 59392",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 59392",
+ "mov ecx, 256",
+ "256:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 256b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1184], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1185], al",
+ "mov rdi, rbx",
+ "add rdi, 1152",
+ "mov rsi, rbx",
+ "add rsi, 60416",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 60416",
+ "mov ecx, 256",
+ "257:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 257b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 61440",
+ "mov ecx, 256",
+ "258:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 258b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 62464",
+ "mov ecx, 256",
+ "259:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 259b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 63488",
+ "mov ecx, 256",
+ "260:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 260b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 64512",
+ "mov ecx, 256",
+ "261:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 261b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 65536",
+ "mov ecx, 256",
+ "262:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 262b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 66560",
+ "mov ecx, 256",
+ "263:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 263b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 67584",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 67584",
+ "mov ecx, 256",
+ "264:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 264b",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 68608",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 68608",
+ "mov ecx, 256",
+ "265:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 265b",
+ "mov eax, 8",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 69632",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 69632",
+ "mov ecx, 256",
+ "266:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 266b",
+ "mov eax, 9",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 70656",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 70656",
+ "mov ecx, 256",
+ "267:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 267b",
+ "mov eax, 10",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 71680",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 71680",
+ "mov ecx, 256",
+ "268:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 268b",
+ "mov eax, 11",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 72704",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 72704",
+ "mov ecx, 256",
+ "269:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 269b",
+ "mov eax, 12",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 73728",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 73728",
+ "mov ecx, 256",
+ "270:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 270b",
+ "mov eax, 13",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 74752",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 74752",
+ "mov ecx, 256",
+ "271:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 271b",
+ "mov eax, 14",
+ "mov BYTE PTR [rbx+1280], al",
+ "mov rdi, rbx",
+ "add rdi, 1216",
+ "mov esi, 2",
+ "mov rdx, rbx",
+ "add rdx, 75776",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "call {vg_mldsa_rej_bounded_poly}",
+ "and r15d, eax",
+ "mov r8d, 0",
+ "sub r8d, eax",
+ "mov rdi, rbx",
+ "add rdi, 75776",
+ "mov ecx, 256",
+ "272:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, r8d",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 272b",
+ "mov rdi, r12",
+ "add rdi, 0",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "273:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 273b",
+ "mov rdi, r13",
+ "add rdi, 0",
+ "mov rsi, rbx",
+ "add rsi, 1024",
+ "mov ecx, 32",
+ "274:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 274b",
+ "mov rdi, r13",
+ "add rdi, 32",
+ "mov rsi, rbx",
+ "add rsi, 1120",
+ "mov ecx, 32",
+ "275:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 275b",
+ "mov rdi, rbx",
+ "add rdi, 61440",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 128",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 62464",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 224",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 63488",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 320",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 64512",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 416",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 65536",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 512",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 66560",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 608",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 67584",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 704",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 68608",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 800",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 69632",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 896",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 70656",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 992",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 71680",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1088",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 72704",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1184",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 73728",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1280",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 74752",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1376",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 75776",
+ "mov esi, 2",
+ "mov edx, 2",
+ "mov rcx, r13",
+ "add rcx, 1472",
+ "mov r8d, 96",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 61440",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 62464",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 63488",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 64512",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 65536",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 66560",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 67584",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 7168",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 9216",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 10240",
+ "mov rdx, rbx",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 68608",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 32",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 1568",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 11264",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 12288",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 13312",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 14336",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 15360",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 16384",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 17408",
+ "mov rdx, rbx",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 69632",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 352",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 1984",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 18432",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 19456",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 20480",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 21504",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 22528",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "mov rdx, rbx",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 672",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 2400",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 27648",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 992",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 2816",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 34816",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 35840",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 1312",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 3232",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 41984",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 43008",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 44032",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 73728",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 1632",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 3648",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 49152",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 50176",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 51200",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 52224",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 74752",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 1952",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 4064",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 57344",
+ "mov rdx, rbx",
+ "add rdx, 64512",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 58368",
+ "mov rdx, rbx",
+ "add rdx, 65536",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 59392",
+ "mov rdx, rbx",
+ "add rdx, 66560",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 60416",
+ "mov rdx, rbx",
+ "add rdx, 67584",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 75776",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 76800",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 78848",
+ "call {vg_mldsa_power2round}",
+ "mov rdi, rbx",
+ "add rdi, 77824",
+ "mov esi, 1023",
+ "mov rdx, r12",
+ "add rdx, 2272",
+ "mov ecx, 320",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 78848",
+ "mov esi, 4095",
+ "mov edx, 4096",
+ "mov rcx, r13",
+ "add rcx, 4480",
+ "mov r8d, 416",
+ "call {vg_mldsa_bit_pack}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 2592",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 8",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r13",
+ "add rcx, 64",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly,
+ vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2,
+ vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ )
+}
+
+/// The CPU features `vg_mldsa87_sign_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA87_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-87 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
+///
+/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness.
+///
+/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address.
+///
+/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing.
+///
+/// # Safety
+///
+/// * `sk` must be valid for reads of 4896 bytes.
+/// * `mu` must be valid for reads of 64 bytes.
+/// * `rnd` must be valid for reads of 32 bytes.
+/// * `sig` must be valid for reads and writes of 4627 bytes.
+/// * `scratch` must be valid for reads and writes of 144384 bytes.
+/// * `sk` must have been written by `vg_mldsa87_keygen`.
+/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do).
+/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 4627], scratch: *mut [u64; 18048]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [r8+840], rbx",
+ "mov QWORD PTR [r8+848], rbp",
+ "mov QWORD PTR [r8+856], r12",
+ "mov QWORD PTR [r8+864], r13",
+ "mov QWORD PTR [r8+872], r14",
+ "mov QWORD PTR [r8+880], r15",
+ "mov rbx, r8",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r14, rcx",
+ "mov r15d, 1",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbp",
+ "add rsi, 0",
+ "mov ecx, 32",
+ "20:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 20b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 64512",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 65536",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 66560",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 67584",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 68608",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 69632",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 73728",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 74752",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 75776",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 76800",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 78848",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 79872",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 80896",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 81920",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 82944",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 83968",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 84992",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 86016",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 87040",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 88064",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 89088",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 90112",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 91136",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 92160",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 93184",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 94208",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 96256",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 97280",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 98304",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 99328",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 100352",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 101376",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 102400",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 103424",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 104448",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 105472",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 106496",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 107520",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 108544",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 109568",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 110592",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 111616",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 112640",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 113664",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 114688",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 115712",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 116736",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 117760",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 118784",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 119808",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+944], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+945], al",
+ "mov rdi, rbx",
+ "add rdi, 912",
+ "mov rsi, rbx",
+ "add rsi, 120832",
+ "mov rdx, rbx",
+ "add rdx, 3072",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "test r15d, r15d",
+ "jne 21f",
+ "jmp 22f",
+ "21:",
+ "mov rdi, rbp",
+ "add rdi, 128",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 40960",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 40960",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 224",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 41984",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 41984",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 320",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 43008",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 43008",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 416",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 44032",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 44032",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 512",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 45056",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 45056",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 608",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 46080",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 704",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 47104",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 800",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 48128",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 896",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 49152",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 49152",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 992",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 50176",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 50176",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1088",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 51200",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 51200",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1184",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 52224",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 52224",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1280",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 53248",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 53248",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1376",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 54272",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 54272",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1472",
+ "mov esi, 96",
+ "mov edx, 2",
+ "mov ecx, 2",
+ "mov r8, rbx",
+ "add r8, 55296",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 55296",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1568",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 56320",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 56320",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1984",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 57344",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 57344",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 2400",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 58368",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 58368",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 2816",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 59392",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 59392",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 3232",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 60416",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 60416",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 3648",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 61440",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 61440",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 4064",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 62464",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 62464",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 4480",
+ "mov esi, 416",
+ "mov edx, 4095",
+ "mov ecx, 4096",
+ "mov r8, rbx",
+ "add r8, 63488",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 63488",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbp",
+ "add rcx, 32",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 32",
+ "mov rcx, r13",
+ "add rcx, 0",
+ "mov r8d, 32",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 128",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 960",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx+896], rax",
+ "mov eax, 814",
+ "mov QWORD PTR [rbx+888], rax",
+ "23:",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 0",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 18432",
+ "mov ecx, 1024",
+ "24:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 24b",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 1",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 19456",
+ "mov ecx, 1024",
+ "25:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 2",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 20480",
+ "mov ecx, 1024",
+ "26:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 26b",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 3",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 21504",
+ "mov ecx, 1024",
+ "27:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 27b",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 4",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 22528",
+ "mov ecx, 1024",
+ "28:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 28b",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 5",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 23552",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov ecx, 1024",
+ "29:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 29b",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 6",
+ "mov BYTE PTR [rbx+1024], al",
+ "shr rax, 8",
+ "mov BYTE PTR [rbx+1025], al",
+ "mov rdi, rbx",
+ "add rdi, 960",
+ "mov esi, 524288",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "mov rcx, rbx",
+ "add rcx, 3072",
+ "call {vg_mldsa_expand_mask_poly}",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "mov ecx, 1024",
+ "210:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 210b",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 64512",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 65536",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 66560",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 67584",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 68608",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 69632",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 73728",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 74752",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 75776",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 76800",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 78848",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 79872",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 80896",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 81920",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 82944",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 83968",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 84992",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 86016",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 87040",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 88064",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 89088",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 90112",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 91136",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 92160",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 93184",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 94208",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 95232",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 96256",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 97280",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 98304",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 99328",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 100352",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 101376",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 102400",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 103424",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 104448",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 105472",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 106496",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 107520",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 108544",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 109568",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 110592",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 111616",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 112640",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 113664",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 114688",
+ "mov rdx, rbx",
+ "add rdx, 25600",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 115712",
+ "mov rdx, rbx",
+ "add rdx, 26624",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 116736",
+ "mov rdx, rbx",
+ "add rdx, 27648",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 117760",
+ "mov rdx, rbx",
+ "add rdx, 28672",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 118784",
+ "mov rdx, rbx",
+ "add rdx, 29696",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 119808",
+ "mov rdx, rbx",
+ "add rdx, 30720",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 120832",
+ "mov rdx, rbx",
+ "add rdx, 31744",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2048",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2176",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2304",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2432",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2560",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2688",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2816",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 6144",
+ "call {vg_mldsa_high_bits}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 2944",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "mov r8d, 1024",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 1040",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov rdi, rbx",
+ "add rdi, 1040",
+ "mov esi, 64",
+ "mov edx, 60",
+ "mov rcx, rbx",
+ "add rcx, 5120",
+ "mov r8, rbx",
+ "add r8, 3072",
+ "call {vg_mldsa_sample_in_ball}",
+ "test eax, eax",
+ "jne 211f",
+ "mov r15d, 0",
+ "mov eax, 1",
+ "mov QWORD PTR [rbx+888], rax",
+ "jmp 212f",
+ "211:",
+ "mov rdi, rbx",
+ "add rdi, 5120",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov r15d, 1",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx+904], rax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 40960",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 41984",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 43008",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 44032",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 45056",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 46080",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 47104",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 48128",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261768",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 49152",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261768",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 50176",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261768",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 51200",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261768",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 52224",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261768",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 53248",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261768",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 54272",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261768",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 55296",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 6144",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 6144",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov esi, 261888",
+ "mov rdx, rbx",
+ "add rdx, 7168",
+ "call {vg_mldsa_low_bits}",
+ "mov rdi, rbx",
+ "add rdi, 7168",
+ "mov esi, 261768",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 56320",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov ecx, 1024",
+ "213:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 213b",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 10240",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 57344",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "mov ecx, 1024",
+ "214:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 214b",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 11264",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 58368",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 34816",
+ "mov ecx, 1024",
+ "215:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 215b",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 34816",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 34816",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 12288",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 59392",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 35840",
+ "mov ecx, 1024",
+ "216:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 216b",
+ "mov rdi, rbx",
+ "add rdi, 35840",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 35840",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 35840",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 13312",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 60416",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov ecx, 1024",
+ "217:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 217b",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 14336",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 61440",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov ecx, 1024",
+ "218:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 218b",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 15360",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 62464",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov ecx, 1024",
+ "219:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 219b",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 16384",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 5120",
+ "mov rdx, rbx",
+ "add rdx, 63488",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 3072",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov esi, 261888",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov ecx, 1024",
+ "220:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 220b",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov rsi, rbx",
+ "add rsi, 8192",
+ "call {vg_mldsa_add_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 17408",
+ "call {vg_mldsa_make_hint}",
+ "mov ecx, DWORD PTR [rbx+904]",
+ "add ecx, eax",
+ "mov QWORD PTR [rbx+904], rcx",
+ "mov eax, DWORD PTR [rbx+904]",
+ "sub rax, 76",
+ "shr rax, 63",
+ "and r15d, eax",
+ "test r15d, r15d",
+ "jne 221f",
+ "mov rax, QWORD PTR [rbx+896]",
+ "add rax, 7",
+ "mov QWORD PTR [rbx+896], rax",
+ "jmp 222f",
+ "221:",
+ "mov eax, 1",
+ "mov QWORD PTR [rbx+888], rax",
+ "222:",
+ "212:",
+ "mov rax, QWORD PTR [rbx+888]",
+ "sub rax, 1",
+ "mov QWORD PTR [rbx+888], rax",
+ "jne 23b",
+ "test r15d, r15d",
+ "jne 223f",
+ "jmp 224f",
+ "223:",
+ "mov rdi, r14",
+ "add rdi, 0",
+ "mov rsi, rbx",
+ "add rsi, 1040",
+ "mov ecx, 64",
+ "225:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 225b",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 64",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 704",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 1344",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 1984",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 2624",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 3264",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov esi, 524287",
+ "mov edx, 524288",
+ "mov rcx, r14",
+ "add rcx, 3904",
+ "mov r8d, 640",
+ "call {vg_mldsa_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 10240",
+ "mov esi, 2048",
+ "mov edx, 75",
+ "mov rcx, r14",
+ "add rcx, 4544",
+ "mov r8d, 83",
+ "call {vg_mldsa_hint_bit_pack}",
+ "224:",
+ "22:",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_high_bits = sym super::mldsa::vg_mldsa_high_bits,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball,
+ vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2,
+ vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt,
+ vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2,
+ vg_mldsa_low_bits = sym super::mldsa::vg_mldsa_low_bits,
+ vg_mldsa_make_hint = sym super::mldsa::vg_mldsa_make_hint,
+ vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack,
+ vg_mldsa_hint_bit_pack = sym super::mldsa::vg_mldsa_hint_bit_pack,
+ )
+}
+
+/// The CPU features `vg_mldsa87_verify_avx2` requires (`Artifact.features`).
+pub(crate) const VG_MLDSA87_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"];
+
+/// ML-DSA-87 verification of a signature of a message representative, `ML-DSA.Verify_internal(pk, M′, σ)` (FIPS 204 Algorithm 8) with `μ` computed by the caller: with the public key `*pk`, the 64-byte message representative `μ = H(H(pk, 64) ‖ M′, 64)` at `mu` and the signature `*sig`, returns 1 if the signature is valid, and 0 if it is not or if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less).
+///
+/// Contract: `VG.Spec.MlDsa.verifyContract`. Not constant time: timing may depend on the public key, the message representative and the signature.
+///
+/// It calls the `vg_mldsa_*` primitives and the SHAKE256 sponge. The samplers' results are combined without a branch, so the only branches depend on the public key and the signature.
+///
+/// # Safety
+///
+/// * `pk` must be valid for reads of 2592 bytes.
+/// * `mu` must be valid for reads of 64 bytes.
+/// * `sig` must be valid for reads of 4627 bytes.
+/// * `scratch` must be valid for reads and writes of 144384 bytes.
+/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3).
+/// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do).
+/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does).
+/// * The CPU must support the `avx` and `avx2` target features.
+#[unsafe(naked)]
+pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592], mu: *const [u8; 64], sig: *const [u8; 4627], scratch: *mut [u64; 18048]) -> u32 {
+ core::arch::naked_asm!(
+ "mov QWORD PTR [rcx+840], rbx",
+ "mov QWORD PTR [rcx+848], rbp",
+ "mov QWORD PTR [rcx+856], r12",
+ "mov QWORD PTR [rcx+864], r13",
+ "mov QWORD PTR [rcx+872], r14",
+ "mov QWORD PTR [rcx+880], r15",
+ "mov rbx, rcx",
+ "mov rbp, rdi",
+ "mov r12, rsi",
+ "mov r13, rdx",
+ "mov r15d, 1",
+ "mov rdi, r13",
+ "add rdi, 4544",
+ "mov esi, 83",
+ "mov edx, 75",
+ "mov rcx, rbx",
+ "add rcx, 8192",
+ "mov r8d, 2048",
+ "call {vg_mldsa_hint_bit_unpack}",
+ "mov r15d, eax",
+ "test r15d, r15d",
+ "jne 20f",
+ "jmp 21f",
+ "20:",
+ "mov rdi, r13",
+ "add rdi, 64",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 16384",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 704",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 17408",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 1344",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 18432",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 1984",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 19456",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 2624",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 20480",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 3264",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 21504",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "mov rdi, r13",
+ "add rdi, 3904",
+ "mov esi, 640",
+ "mov edx, 524287",
+ "mov ecx, 524288",
+ "mov r8, rbx",
+ "add r8, 22528",
+ "call {vg_mldsa_bit_unpack}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov esi, 524168",
+ "call {vg_mldsa_norm_lt}",
+ "and r15d, eax",
+ "test r15d, r15d",
+ "jne 22f",
+ "jmp 23f",
+ "22:",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbp",
+ "add rsi, 0",
+ "mov ecx, 32",
+ "24:",
+ "movzx eax, BYTE PTR [rsi]",
+ "mov BYTE PTR [rdi], al",
+ "add rdi, 1",
+ "add rsi, 1",
+ "sub rcx, 1",
+ "jne 24b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 28672",
+ "mov ecx, 256",
+ "25:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 25b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 29696",
+ "mov ecx, 256",
+ "26:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 26b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 30720",
+ "mov ecx, 256",
+ "27:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 27b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 31744",
+ "mov ecx, 256",
+ "28:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 28b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 32768",
+ "mov ecx, 256",
+ "29:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 29b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 33792",
+ "mov ecx, 256",
+ "210:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 210b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 34816",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 34816",
+ "mov ecx, 256",
+ "211:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 211b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 36864",
+ "mov ecx, 256",
+ "212:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 212b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 37888",
+ "mov ecx, 256",
+ "213:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 213b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 38912",
+ "mov ecx, 256",
+ "214:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 214b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 39936",
+ "mov ecx, 256",
+ "215:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 215b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 40960",
+ "mov ecx, 256",
+ "216:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 216b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 41984",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 41984",
+ "mov ecx, 256",
+ "217:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 217b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 43008",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 43008",
+ "mov ecx, 256",
+ "218:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 218b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 45056",
+ "mov ecx, 256",
+ "219:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 219b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 46080",
+ "mov ecx, 256",
+ "220:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 220b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 47104",
+ "mov ecx, 256",
+ "221:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 221b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 48128",
+ "mov ecx, 256",
+ "222:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 222b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 49152",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 49152",
+ "mov ecx, 256",
+ "223:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 223b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 50176",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 50176",
+ "mov ecx, 256",
+ "224:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 224b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 51200",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 51200",
+ "mov ecx, 256",
+ "225:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 225b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 53248",
+ "mov ecx, 256",
+ "226:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 226b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 54272",
+ "mov ecx, 256",
+ "227:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 227b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 55296",
+ "mov ecx, 256",
+ "228:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 228b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 56320",
+ "mov ecx, 256",
+ "229:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 229b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 57344",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 57344",
+ "mov ecx, 256",
+ "230:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 230b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 58368",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 58368",
+ "mov ecx, 256",
+ "231:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 231b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 59392",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 59392",
+ "mov ecx, 256",
+ "232:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 232b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 61440",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 61440",
+ "mov ecx, 256",
+ "233:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 233b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 62464",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 62464",
+ "mov ecx, 256",
+ "234:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 234b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 63488",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 63488",
+ "mov ecx, 256",
+ "235:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 235b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 64512",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 64512",
+ "mov ecx, 256",
+ "236:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 236b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 65536",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 65536",
+ "mov ecx, 256",
+ "237:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 237b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 66560",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 66560",
+ "mov ecx, 256",
+ "238:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 238b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 67584",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 67584",
+ "mov ecx, 256",
+ "239:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 239b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 69632",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 69632",
+ "mov ecx, 256",
+ "240:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 240b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 70656",
+ "mov ecx, 256",
+ "241:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 241b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 71680",
+ "mov ecx, 256",
+ "242:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 242b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 72704",
+ "mov ecx, 256",
+ "243:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 243b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 73728",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 73728",
+ "mov ecx, 256",
+ "244:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 244b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 74752",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 74752",
+ "mov ecx, 256",
+ "245:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 245b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 75776",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 75776",
+ "mov ecx, 256",
+ "246:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 246b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 77824",
+ "mov ecx, 256",
+ "247:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 247b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 78848",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 78848",
+ "mov ecx, 256",
+ "248:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 248b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 79872",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 79872",
+ "mov ecx, 256",
+ "249:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 249b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 80896",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 80896",
+ "mov ecx, 256",
+ "250:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 250b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 81920",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 81920",
+ "mov ecx, 256",
+ "251:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 251b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 82944",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 82944",
+ "mov ecx, 256",
+ "252:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 252b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 83968",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 83968",
+ "mov ecx, 256",
+ "253:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 253b",
+ "mov eax, 0",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 86016",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 86016",
+ "mov ecx, 256",
+ "254:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 254b",
+ "mov eax, 1",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 87040",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 87040",
+ "mov ecx, 256",
+ "255:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 255b",
+ "mov eax, 2",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 88064",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 88064",
+ "mov ecx, 256",
+ "256:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 256b",
+ "mov eax, 3",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 89088",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 89088",
+ "mov ecx, 256",
+ "257:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 257b",
+ "mov eax, 4",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 90112",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 90112",
+ "mov ecx, 256",
+ "258:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 258b",
+ "mov eax, 5",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 91136",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 91136",
+ "mov ecx, 256",
+ "259:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 259b",
+ "mov eax, 6",
+ "mov BYTE PTR [rbx+928], al",
+ "mov eax, 7",
+ "mov BYTE PTR [rbx+929], al",
+ "mov rdi, rbx",
+ "add rdi, 896",
+ "mov rsi, rbx",
+ "add rsi, 92160",
+ "mov rdx, rbx",
+ "add rdx, 4096",
+ "call {vg_mldsa_rej_ntt_poly}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 92160",
+ "mov ecx, 256",
+ "260:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 260b",
+ "mov rdi, r13",
+ "add rdi, 0",
+ "mov esi, 64",
+ "mov edx, 60",
+ "mov rcx, rbx",
+ "add rcx, 23552",
+ "mov r8, rbx",
+ "add r8, 4096",
+ "call {vg_mldsa_sample_in_ball}",
+ "and r15d, eax",
+ "mov edx, 0",
+ "sub edx, eax",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov ecx, 256",
+ "261:",
+ "mov eax, DWORD PTR [rdi]",
+ "and eax, edx",
+ "mov DWORD PTR [rdi], eax",
+ "add rdi, 4",
+ "sub rcx, 1",
+ "jne 261b",
+ "mov rdi, rbx",
+ "add rdi, 16384",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 17408",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 18432",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 19456",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 20480",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 21504",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 22528",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 23552",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 28672",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 29696",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 30720",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 31744",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 32768",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 33792",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 34816",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 32",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 8192",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1024",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 36864",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 37888",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 38912",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 39936",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 40960",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 41984",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 43008",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 352",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 9216",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1152",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 45056",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 46080",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 47104",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 48128",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 49152",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 50176",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 51200",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 672",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 10240",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1280",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 53248",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 54272",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 55296",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 56320",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 57344",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 58368",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 59392",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 992",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 11264",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1408",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 61440",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 62464",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 63488",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 64512",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 65536",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 66560",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 67584",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1312",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 12288",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1536",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 69632",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 70656",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 71680",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 72704",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 73728",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 74752",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 75776",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1632",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 13312",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1664",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 77824",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 78848",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 79872",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 80896",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 81920",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 82944",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 83968",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 1952",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 14336",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1792",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 86016",
+ "mov rdx, rbx",
+ "add rdx, 16384",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 87040",
+ "mov rdx, rbx",
+ "add rdx, 17408",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 88064",
+ "mov rdx, rbx",
+ "add rdx, 18432",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 89088",
+ "mov rdx, rbx",
+ "add rdx, 19456",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 90112",
+ "mov rdx, rbx",
+ "add rdx, 20480",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 91136",
+ "mov rdx, rbx",
+ "add rdx, 21504",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 92160",
+ "mov rdx, rbx",
+ "add rdx, 22528",
+ "call {vg_mldsa_multiply_add_ntt_avx2}",
+ "mov rdi, rbp",
+ "add rdi, 2272",
+ "mov rsi, rbx",
+ "add rsi, 24576",
+ "call {vg_mldsa_unpack_t1}",
+ "mov rdi, rbx",
+ "add rdi, 24576",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 25600",
+ "mov rsi, rbx",
+ "add rsi, 23552",
+ "mov rdx, rbx",
+ "add rdx, 24576",
+ "call {vg_mldsa_multiply_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 25600",
+ "call {vg_mldsa_sub_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 26624",
+ "mov rsi, rbx",
+ "add rsi, 4096",
+ "call {vg_mldsa_inv_ntt_avx2}",
+ "mov rdi, rbx",
+ "add rdi, 15360",
+ "mov rsi, rbx",
+ "add rsi, 26624",
+ "mov edx, 261888",
+ "mov rcx, rbx",
+ "add rcx, 27648",
+ "call {vg_mldsa_use_hint}",
+ "mov rdi, rbx",
+ "add rdi, 27648",
+ "mov esi, 15",
+ "mov rdx, rbx",
+ "add rdx, 1920",
+ "mov ecx, 128",
+ "call {vg_mldsa_simple_bit_pack}",
+ "mov eax, 0",
+ "mov QWORD PTR [rbx], rax",
+ "mov QWORD PTR [rbx+8], rax",
+ "mov QWORD PTR [rbx+16], rax",
+ "mov QWORD PTR [rbx+24], rax",
+ "mov QWORD PTR [rbx+32], rax",
+ "mov QWORD PTR [rbx+40], rax",
+ "mov QWORD PTR [rbx+48], rax",
+ "mov QWORD PTR [rbx+56], rax",
+ "mov QWORD PTR [rbx+64], rax",
+ "mov QWORD PTR [rbx+72], rax",
+ "mov QWORD PTR [rbx+80], rax",
+ "mov QWORD PTR [rbx+88], rax",
+ "mov QWORD PTR [rbx+96], rax",
+ "mov QWORD PTR [rbx+104], rax",
+ "mov QWORD PTR [rbx+112], rax",
+ "mov QWORD PTR [rbx+120], rax",
+ "mov QWORD PTR [rbx+128], rax",
+ "mov QWORD PTR [rbx+136], rax",
+ "mov QWORD PTR [rbx+144], rax",
+ "mov QWORD PTR [rbx+152], rax",
+ "mov QWORD PTR [rbx+160], rax",
+ "mov QWORD PTR [rbx+168], rax",
+ "mov QWORD PTR [rbx+176], rax",
+ "mov QWORD PTR [rbx+184], rax",
+ "mov QWORD PTR [rbx+192], rax",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, r12",
+ "add rcx, 0",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 64",
+ "mov rcx, rbx",
+ "add rcx, 1024",
+ "mov r8d, 1024",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_absorb}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov ecx, 31",
+ "mov r8, rbx",
+ "add r8, 200",
+ "call {vg_keccak_pad}",
+ "mov rdi, rbx",
+ "add rdi, 0",
+ "mov esi, 136",
+ "mov edx, 0",
+ "mov rcx, rbx",
+ "add rcx, 2048",
+ "mov r8d, 64",
+ "mov r9, rbx",
+ "add r9, 200",
+ "call {vg_keccak_squeeze}",
+ "mov rsi, rbx",
+ "add rsi, 2048",
+ "mov rdi, r13",
+ "add rdi, 0",
+ "mov ecx, 64",
+ "mov edx, 0",
+ "262:",
+ "movzx eax, BYTE PTR [rsi]",
+ "movzx r8d, BYTE PTR [rdi]",
+ "xor rax, r8",
+ "or rdx, rax",
+ "add rsi, 1",
+ "add rdi, 1",
+ "sub rcx, 1",
+ "jne 262b",
+ "sub rdx, 1",
+ "sbb rax, rax",
+ "and r15d, eax",
+ "23:",
+ "21:",
+ "mov eax, r15d",
+ "mov r15, QWORD PTR [rbx+880]",
+ "mov r14, QWORD PTR [rbx+872]",
+ "mov r13, QWORD PTR [rbx+864]",
+ "mov r12, QWORD PTR [rbx+856]",
+ "mov rbp, QWORD PTR [rbx+848]",
+ "mov rbx, QWORD PTR [rbx+840]",
+ "ret",
+ vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack,
+ vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack,
+ vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt,
+ vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly,
+ vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball,
+ vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2,
+ vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2,
+ vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2,
+ vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1,
+ vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2,
+ vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2,
+ vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint,
+ vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack,
+ vg_keccak_absorb = sym super::sha3::vg_keccak_absorb,
+ vg_keccak_pad = sym super::sha3::vg_keccak_pad,
+ vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze,
+ )
+}
+
/// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed.
///
/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys.
diff --git a/src/mldsa44.rs b/src/mldsa44.rs
index 782c59c4b..04777dad9 100644
--- a/src/mldsa44.rs
+++ b/src/mldsa44.rs
@@ -38,6 +38,9 @@ crate::mldsa_common::ml_dsa! {
keygen_sha3: (crate::arch::mldsa44::vg_mldsa44_keygen_sha3, crate::arch::mldsa44::VG_MLDSA44_KEYGEN_SHA3_FEATURES),
sign_sha3: (crate::arch::mldsa44::vg_mldsa44_sign_sha3, crate::arch::mldsa44::VG_MLDSA44_SIGN_SHA3_FEATURES),
verify_sha3: (crate::arch::mldsa44::vg_mldsa44_verify_sha3, crate::arch::mldsa44::VG_MLDSA44_VERIFY_SHA3_FEATURES),
+ keygen_avx2: (crate::arch::mldsa44::vg_mldsa44_keygen_avx2, crate::arch::mldsa44::VG_MLDSA44_KEYGEN_AVX2_FEATURES),
+ sign_avx2: (crate::arch::mldsa44::vg_mldsa44_sign_avx2, crate::arch::mldsa44::VG_MLDSA44_SIGN_AVX2_FEATURES),
+ verify_avx2: (crate::arch::mldsa44::vg_mldsa44_verify_avx2, crate::arch::mldsa44::VG_MLDSA44_VERIFY_AVX2_FEATURES),
pk: 1312,
sk: 2560,
sig: 2420,
diff --git a/src/mldsa65.rs b/src/mldsa65.rs
index a67072013..79dff1485 100644
--- a/src/mldsa65.rs
+++ b/src/mldsa65.rs
@@ -38,6 +38,9 @@ crate::mldsa_common::ml_dsa! {
keygen_sha3: (crate::arch::mldsa65::vg_mldsa65_keygen_sha3, crate::arch::mldsa65::VG_MLDSA65_KEYGEN_SHA3_FEATURES),
sign_sha3: (crate::arch::mldsa65::vg_mldsa65_sign_sha3, crate::arch::mldsa65::VG_MLDSA65_SIGN_SHA3_FEATURES),
verify_sha3: (crate::arch::mldsa65::vg_mldsa65_verify_sha3, crate::arch::mldsa65::VG_MLDSA65_VERIFY_SHA3_FEATURES),
+ keygen_avx2: (crate::arch::mldsa65::vg_mldsa65_keygen_avx2, crate::arch::mldsa65::VG_MLDSA65_KEYGEN_AVX2_FEATURES),
+ sign_avx2: (crate::arch::mldsa65::vg_mldsa65_sign_avx2, crate::arch::mldsa65::VG_MLDSA65_SIGN_AVX2_FEATURES),
+ verify_avx2: (crate::arch::mldsa65::vg_mldsa65_verify_avx2, crate::arch::mldsa65::VG_MLDSA65_VERIFY_AVX2_FEATURES),
pk: 1952,
sk: 4032,
sig: 3309,
diff --git a/src/mldsa87.rs b/src/mldsa87.rs
index 7b672b3e4..ce1e72136 100644
--- a/src/mldsa87.rs
+++ b/src/mldsa87.rs
@@ -38,6 +38,9 @@ crate::mldsa_common::ml_dsa! {
keygen_sha3: (crate::arch::mldsa87::vg_mldsa87_keygen_sha3, crate::arch::mldsa87::VG_MLDSA87_KEYGEN_SHA3_FEATURES),
sign_sha3: (crate::arch::mldsa87::vg_mldsa87_sign_sha3, crate::arch::mldsa87::VG_MLDSA87_SIGN_SHA3_FEATURES),
verify_sha3: (crate::arch::mldsa87::vg_mldsa87_verify_sha3, crate::arch::mldsa87::VG_MLDSA87_VERIFY_SHA3_FEATURES),
+ keygen_avx2: (crate::arch::mldsa87::vg_mldsa87_keygen_avx2, crate::arch::mldsa87::VG_MLDSA87_KEYGEN_AVX2_FEATURES),
+ sign_avx2: (crate::arch::mldsa87::vg_mldsa87_sign_avx2, crate::arch::mldsa87::VG_MLDSA87_SIGN_AVX2_FEATURES),
+ verify_avx2: (crate::arch::mldsa87::vg_mldsa87_verify_avx2, crate::arch::mldsa87::VG_MLDSA87_VERIFY_AVX2_FEATURES),
pk: 2592,
sk: 4896,
sig: 4627,
diff --git a/src/mldsa_common.rs b/src/mldsa_common.rs
index 2979826b0..448792854 100644
--- a/src/mldsa_common.rs
+++ b/src/mldsa_common.rs
@@ -25,6 +25,45 @@ pub(crate) fn message_rep(tr: &[u8; 64], msg: &[u8], ctx: &[u8]) -> Option<[u8;
Some(mu)
}
+/// The implementations of ML-DSA's verified functions: their instances for
+/// the polynomial arithmetic and Keccak they call (`Generic/MlDsaArith/` and
+/// the Keccak backend, `crate::hashes::sha3::Backend`).
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum Backend {
+ /// The target's baseline ISA (on x86-64, SSE2 polynomial arithmetic),
+ /// with scalar Keccak.
+ Scalar,
+ /// AArch64 with the SHA-3 extension, for Keccak.
+ #[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))]
+ Sha3,
+ /// x86-64 with AVX2, for the polynomial arithmetic.
+ #[cfg(target_arch = "x86_64")]
+ Avx2,
+}
+
+impl Backend {
+ /// The implementation for the Keccak backend `keccak`, on a CPU with the
+ /// features `f`, of functions whose AVX2 instances need `avx2`.
+ #[cfg_attr(not(target_arch = "x86_64"), allow(unused_variables))]
+ pub(crate) fn select(
+ keccak: crate::hashes::sha3::Backend,
+ f: crate::cpu::Features,
+ avx2: &[&[&str]],
+ ) -> Backend {
+ match keccak {
+ crate::hashes::sha3::Backend::Scalar => {
+ #[cfg(target_arch = "x86_64")]
+ if f.contains(crate::cpu::Features::all(avx2)) {
+ return Backend::Avx2;
+ }
+ Backend::Scalar
+ }
+ #[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))]
+ crate::hashes::sha3::Backend::Sha3 => Backend::Sha3,
+ }
+ }
+}
+
/// Defines the API of one ML-DSA parameter set: its `Error`, `SigningKey`
/// and `VerifyingKey`, over its verified `keygen`, `sign` and `verify`
/// functions (which take `μ`) and its sizes.
@@ -39,6 +78,9 @@ macro_rules! ml_dsa {
keygen_sha3: ($keygen_sha3:path, $keygen_sha3_features:path),
sign_sha3: ($sign_sha3:path, $sign_sha3_features:path),
verify_sha3: ($verify_sha3:path, $verify_sha3_features:path),
+ keygen_avx2: ($keygen_avx2:path, $keygen_avx2_features:path),
+ sign_avx2: ($sign_avx2:path, $sign_avx2_features:path),
+ verify_avx2: ($verify_avx2:path, $verify_avx2_features:path),
pk: $pk:literal,
sk: $sk:literal,
sig: $sig:literal,
@@ -46,10 +88,12 @@ macro_rules! ml_dsa {
) => {
use $crate::mldsa_common::message_rep;
use $crate::zeroize::zeroize;
- use $crate::hashes::sha3::Backend;
+ use $crate::mldsa_common::Backend;
- /// Follow the shared Keccak backend only when all generated callers'
- /// feature requirements are met. The draft's default stays scalar.
+ /// The implementation to call: the Keccak backend, followed only
+ /// when all generated callers' feature requirements are met (the
+ /// draft's default stays scalar), and AVX2 for the polynomial
+ /// arithmetic on x86-64 when the CPU has what its callers need.
fn backend() -> Backend {
#[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))]
if !$crate::cpu::detected().contains($crate::cpu::Features::all(&[
@@ -59,7 +103,11 @@ macro_rules! ml_dsa {
])) {
return Backend::Scalar;
}
- Backend::detected()
+ #[cfg(target_arch = "x86_64")]
+ const AVX2: &[&[&str]] = &[$keygen_avx2_features, $sign_avx2_features, $verify_avx2_features];
+ #[cfg(not(target_arch = "x86_64"))]
+ const AVX2: &[&[&str]] = &[];
+ Backend::select($crate::hashes::sha3::Backend::detected(), $crate::cpu::detected(), AVX2)
}
/// Why an operation failed.
@@ -140,6 +188,8 @@ macro_rules! ml_dsa {
Backend::Scalar => $verify(&self.bytes, mu, sig, &mut scratch),
#[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))]
Backend::Sha3 => $verify_sha3(&self.bytes, mu, sig, &mut scratch),
+ #[cfg(target_arch = "x86_64")]
+ Backend::Avx2 => $verify_avx2(&self.bytes, mu, sig, &mut scratch),
}
};
if r == 1 { Ok(()) } else { Err(Error::InvalidSignature) }
@@ -196,6 +246,8 @@ macro_rules! ml_dsa {
Backend::Scalar => $keygen(seed, &mut key.vk.bytes, &mut key.sk, &mut scratch),
#[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))]
Backend::Sha3 => $keygen_sha3(seed, &mut key.vk.bytes, &mut key.sk, &mut scratch),
+ #[cfg(target_arch = "x86_64")]
+ Backend::Avx2 => $keygen_avx2(seed, &mut key.vk.bytes, &mut key.sk, &mut scratch),
}
};
zeroize(&mut scratch);
@@ -274,6 +326,8 @@ macro_rules! ml_dsa {
Backend::Scalar => $sign(&self.sk, mu, rnd, &mut sig, &mut scratch),
#[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))]
Backend::Sha3 => $sign_sha3(&self.sk, mu, rnd, &mut sig, &mut scratch),
+ #[cfg(target_arch = "x86_64")]
+ Backend::Avx2 => $sign_avx2(&self.sk, mu, rnd, &mut sig, &mut scratch),
}
};
zeroize(&mut scratch);