From c8a898673abe4aefd7f6253bf95e739b94b96c5c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 13:33:05 +0000 Subject: [PATCH 1/8] =?UTF-8?q?ML-DSA=20on=20x86-64:=20NTT=20and=20NTT?= =?UTF-8?q?=E2=81=BB=C2=B9=20in=20SSE2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vg_mldsa_ntt and vg_mldsa_inv_ntt now compute on four coefficients at a time in SSE2 registers, as ML-KEM's x86-64 NTT does: a Montgomery multiplication with pmuludq (the even doublewords, then the odd ones moved down by pshufd), conditional additions of q with psrad masks, and for the layers with len 2 and 1 the coefficients of two or four blocks gathered with punpck{l,h}qdq (and pshufd) and interleaved back. The zetas are a table in Montgomery form that the prologue stores in scratch; the multiplications run inside ML-KEM's withMxcsr, so Intel's MCDT mitigation holds. Proofs: the lanes' arithmetic (VArith), the butterflies on registers (VLanes), the loads and stores of four coefficients and the zetas (VMem), the layers (VLay, VLay21), and the functions (Ntt, NttInv). Signing and verification now check that their code loads MXCSR only to restore it (ctlOk, through a compositional ctlC for verification) rather than never, since their primitives now do. ML-DSA-65 on this machine: sign 1.53 ms -> 0.81 ms, verify 281 us -> 181 us, keygen 276 us -> 248 us. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- README.md | 6 +- docs/algorithms/ml-dsa-44.toml | 1 + docs/algorithms/ml-dsa-65.toml | 1 + docs/algorithms/ml-dsa-87.toml | 1 + .../Artifacts/MlDsaArith/X86_64.lean | 10 +- .../Impl/MlDsa/X86_64/Arith/Common.lean | 12 +- .../Impl/MlDsa/X86_64/Arith/Ntt.lean | 177 +- .../Impl/MlDsa/X86_64/Arith/Vec.lean | 88 + .../Proof/Framework/X86_64/Mxcsr.lean | 25 + .../Proof/MlDsa/X86_64/Arith/Ntt.lean | 323 +- .../Proof/MlDsa/X86_64/Arith/NttBfly.lean | 201 -- .../Proof/MlDsa/X86_64/Arith/NttInv.lean | 327 +- .../Proof/MlDsa/X86_64/Arith/NttLoop.lean | 165 - .../Proof/MlDsa/X86_64/Arith/Table.lean | 37 +- .../Proof/MlDsa/X86_64/Arith/VArith.lean | 285 ++ .../Proof/MlDsa/X86_64/Arith/VLanes.lean | 138 + .../Proof/MlDsa/X86_64/Arith/VLay.lean | 322 ++ .../Proof/MlDsa/X86_64/Arith/VLay21.lean | 458 +++ .../Proof/MlDsa/X86_64/Arith/VMem.lean | 104 + .../Proof/MlDsa/X86_64/Sign/Correct.lean | 5 +- .../Proof/MlDsa/X86_64/Sign/Verified.lean | 2 +- .../Proof/MlDsa/X86_64/Verify/Correct.lean | 2 +- .../Proof/MlDsa/X86_64/Verify/Entry.lean | 9 +- .../Proof/MlDsa/X86_64/Verify/Instrs.lean | 85 +- src/asm/x86_64/mldsa.rs | 2920 ++++++++--------- 25 files changed, 3323 insertions(+), 2381 deletions(-) create mode 100644 lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Vec.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttBfly.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttLoop.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VArith.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLanes.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLay.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLay21.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VMem.lean diff --git a/README.md b/README.md index c4ed49c34..00030ddd4 100644 --- a/README.md +++ b/README.md @@ -774,7 +774,7 @@ yours to keep: ✅ -✅ +✅ SSE2 NTT ✅ SHA extensions @@ -790,7 +790,7 @@ yours to keep: ✅ -✅ +✅ SSE2 NTT ✅ SHA extensions @@ -806,7 +806,7 @@ yours to keep: ✅ -✅ +✅ SSE2 NTT ✅ SHA extensions diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml index b87af90f8..7f316830d 100644 --- a/docs/algorithms/ml-dsa-44.toml +++ b/docs/algorithms/ml-dsa-44.toml @@ -3,3 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa44.rs"] asm = ["mldsa44", "mldsa"] +optimized = { x86_64 = "SSE2 NTT" } diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml index 1220edd86..26de285e2 100644 --- a/docs/algorithms/ml-dsa-65.toml +++ b/docs/algorithms/ml-dsa-65.toml @@ -3,3 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa65.rs"] asm = ["mldsa65", "mldsa"] +optimized = { x86_64 = "SSE2 NTT" } diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml index d463e224c..1f76a641b 100644 --- a/docs/algorithms/ml-dsa-87.toml +++ b/docs/algorithms/ml-dsa-87.toml @@ -3,3 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa87.rs"] asm = ["mldsa87", "mldsa"] +optimized = { x86_64 = "SSE2 NTT" } diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean index 8cd4766b7..f8fdd427a 100644 --- a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean +++ b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean @@ -22,7 +22,10 @@ def artifacts : List Artifact := [ { Spec.MlDsa.nttApi with target := X86_64.target doc := Spec.MlDsa.nttApi.doc - (notes := ["The function stores a table of the 256 zetas in `scratch`."]) + (notes := ["The function computes on four coefficients at a time in SSE2 registers, with a table of \ + the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications \ + (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back \ + before returning."]) code := Impl.MlDsa.X86_64.Arith.ntt contract := Spec.MlDsa.nttContract X86_64.abi verified := Proof.MlDsa.X86_64.Arith.ntt_verified @@ -31,7 +34,10 @@ def artifacts : List Artifact := [ { Spec.MlDsa.nttInvApi with target := X86_64.target doc := Spec.MlDsa.nttInvApi.doc - (notes := ["The function stores a table of the 256 negated zetas in `scratch`."]) + (notes := ["The function computes on four coefficients at a time in SSE2 registers, with a table of \ + the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications \ + (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back \ + before returning."]) code := Impl.MlDsa.X86_64.Arith.nttInv contract := Spec.MlDsa.nttInvContract X86_64.abi verified := Proof.MlDsa.X86_64.Arith.nttInv_verified diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Common.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Common.lean index e808dbeac..ca4ccf001 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Common.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Common.lean @@ -16,10 +16,7 @@ Pieces of code that the ML-DSA arithmetic functions share, for `⌊rax / q⌋ - 1`, so `rax` less that quotient times `q` (a second `mul`) is less than `2q`, and `csubQ` reduces it. `mul` is the only multiplication of the model, and its timing does not depend on its operands (it is on - Intel's DOIT list). It uses `rax`, `rdx` and `r11`; -* `storeTab t n`: the table `t 0, …, t (n - 1)` of constants stored as - `u32`s at `r9` (in the working space: the code has no other memory), with - immediates. It uses `rax`. + Intel's DOIT list). It uses `rax`, `rdx` and `r11`. -/ namespace VG.Impl.MlDsa.X86_64.Arith @@ -47,11 +44,4 @@ def reduce : List Instr := [.mov .r10 (.reg .rax), .movImm64 .r11 barrettImm, .mul .r11, .mov .rax (.reg .rdx), .mov .r11 (.imm qImm), .mul .r11, .alu .sub .r10 (.reg .rax)] ++ csubQ .r10 .r11 -/-- `t i` to `[r9 + 4i]`. -/ -def tabStep (t : Nat → Nat) (i : Nat) : List Instr := - [.mov32 .rax (.imm (BitVec.ofNat 32 (t i))), .store32 (at_ .r9 (4 * i)) .rax] - -/-- The table `t 0, …, t (n - 1)` at `r9`. -/ -def storeTab (t : Nat → Nat) (n : Nat) : List Instr := (List.range n).flatMap (tabStep t) - end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Ntt.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Ntt.lean index 6c0808ddf..97f85c3ad 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Ntt.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Ntt.lean @@ -1,97 +1,104 @@ -import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Common +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Vec import VerifiedGarbage.Spec.MlDsa /-! # ML-DSA on x86-64: `vg_mldsa_ntt` and `vg_mldsa_inv_ntt` -`ntt(f = rdi, scratch = rsi)` and `nttInv(f = rdi, scratch = rsi)`: the -prologue stores a table of 256 zetas to `scratch` as `u32`s (`storeTab`, -with `r9` = `scratch`): `ζ^BitRev8(m) mod q` for `NTT`, and its negation -`-ζ^BitRev8(m) mod q` for `NTT⁻¹` (the `z` of Algorithm 42). Then each of -the eight layers runs its blocks, with `rsi` pointing at coefficient `j` of -`f`, `r8` at the zeta of the block, `rdi` counting the blocks down and `rcx` -the butterflies of a block; the zeta of the block is in `r9`. - -* `NTT` (Algorithm 41): the layers with `len` = 128, 64, …, 1, whose zetas - are consecutive, from `m = 1` up. A butterfly computes - `t = ζ · f[j + len] mod q` (with `reduce`, a Barrett reduction with - `mul`), and stores `f[j] - t` (`f[j] + q - t`, reduced with `csubQ`) to - `f[j + len]` and `f[j] + t` (reduced) to `f[j]`. -* `NTT⁻¹` (Algorithm 42): the layers with `len` = 1, 2, …, 128, whose zetas - are consecutive from `m = 255` down. A butterfly stores `f[j] + f[j + len]` - (reduced) to `f[j]` and `z · (f[j] - f[j + len]) mod q` to `f[j + len]`. - Then every coefficient is multiplied by `8347681 = 256⁻¹ mod q` and - reduced. - -A block ends with `rsi` advanced past its upper half, so a layer ends with -`rsi` at `f + 1024`, and moves it back. Every address and branch depends -only on the pointers. +`ntt(f = rdi, scratch = rsi)` and `nttInv(f = rdi, scratch = rsi)` compute +on four coefficients of `f` at a time, in place, as doublewords of SSE +registers (see `Vec.lean`). `scratch` holds the 256 `u32`s +`ζ^BitRev8(m) · 2³² mod q` (`zmTab`, stored with immediates: the code has +no other memory), and MXCSR's at bytes 768 to 775 before and after +(`withMxcsr`, which saves the caller's MXCSR in `r11` in between). + +Within `withMxcsr`, the prologue stores the table and the constants. Then +the layers, each a pass over `f` with `rdx` pointing at the coefficients it +loads and `r8` at the zetas of the table: + +* `NTT` (Algorithm 41): the layers with `len` = 128, 64, 32, 16, 8 and 4 + (`vlay`) run their blocks (counted in `rax`), each its zeta in the + doublewords of `xmm13` (`vzeta`), and `len / 4` times the butterflies of + four coefficients `w[j]` and of the four `w[j + len]` (`vbfly`, counted + in `rcx`). The layer with `len = 2` (`vlay2`) loads the 8 coefficients of + two blocks, gathers their lower and upper halves into `xmm0` and `xmm1` + (`punpcklqdq`, `punpckhqdq`), with the two zetas in the halves of + `xmm13`; the layer with `len = 1` (`vlay1`) those of four blocks, their + pairs gathered (with `pshufd` first), with the four zetas in the + doublewords of `xmm13`. +* `NTT⁻¹` (Algorithm 42): the same layers in the opposite order, with the + zetas from `m = 255` down and the inverse butterflies (`vibfly`); then + every coefficient is multiplied by `8347681 = 256⁻¹ mod q` (`vscale`, + with `vmont` by `8347681 · 2³² mod q = 16382`). + +Every address and branch depends only on the pointers. -/ namespace VG.Impl.MlDsa.X86_64.Arith open VG.X86_64 - -/-- `ζ^BitRev8(m) mod q`. -/ -def zetaTab (m : Nat) : Nat := 1753 ^ Spec.MlDsa.bitRev8 m % 8380417 - -/-- `-ζ^BitRev8(m) mod q`. -/ -def negZetaTab (m : Nat) : Nat := (8380417 - zetaTab m) % 8380417 - -/-- A butterfly of `NTT` on `[rsi]` and `[rsi + 4len]`, with the zeta in `r9`. -/ -def bfly (len : Nat) : List Instr := - [.mov32 .rax (.mem (at_ .rsi (4 * len))), .mul .r9] ++ reduce ++ - [.mov32 .rax (.mem (at_ .rsi 0)), .mov32 .rdx (.reg .rax), .alu32 .add .rdx (.imm qImm), - .alu32 .sub .rdx (.reg .r10)] ++ csubQ .rdx .r11 ++ - [.store32 (at_ .rsi (4 * len)) .rdx, .alu32 .add .rax (.reg .r10)] ++ csubQ .rax .r11 ++ - [.store32 (at_ .rsi 0) .rax, .alu .add .rsi (.imm 4), .alu .sub .rcx (.imm 1)] - -/-- A butterfly of `NTT⁻¹` on `[rsi]` and `[rsi + 4len]`, with the zeta in `r9`. -/ -def bflyInv (len : Nat) : List Instr := - [.mov32 .rax (.mem (at_ .rsi 0)), .mov32 .r10 (.mem (at_ .rsi (4 * len))), .mov32 .rdx (.reg .rax), - .alu32 .add .rdx (.reg .r10)] ++ csubQ .rdx .r11 ++ - [.store32 (at_ .rsi 0) .rdx, .alu32 .add .rax (.imm qImm), .alu32 .sub .rax (.reg .r10)] ++ - csubQ .rax .r11 ++ [.mul .r9] ++ reduce ++ - [.store32 (at_ .rsi (4 * len)) .r10, .alu .add .rsi (.imm 4), .alu .sub .rcx (.imm 1)] - -/-- A block of `len` butterflies `b`, with the zeta at `r8`, which then moves -by `dz` bytes (4 or -4). -/ -def nttBlk (b : List Instr) (len : Nat) (dz : BitVec 32) : Prog isa := - .seq (.block [.mov32 .r9 (.mem (at_ .r8 0)), .alu .add .r8 (.imm dz), .mov32 .rcx (.imm (BitVec.ofNat 32 len))]) - (.seq (.loop (.block b) .ne) - (.block [.alu .add .rsi (.imm (BitVec.ofNat 32 (4 * len))), .alu .sub .rdi (.imm 1)])) - -/-- A layer: its `128 / len` blocks, then `rsi` back to `f`. -/ -def nttLay (b : List Instr) (len : Nat) (dz : BitVec 32) : Prog isa := - .seq (.block [.mov32 .rdi (.imm (BitVec.ofNat 32 (128 / len)))]) - (.seq (.loop (nttBlk b len dz) .ne) (.block [.alu .sub .rsi (.imm 1024)])) - -/-- The layers of `NTT` with `len` in `lens`. -/ -def nttLays : List Nat → Prog isa - | [] => .block [] - | len :: lens => .seq (nttLay (bfly len) len 4) (nttLays lens) - -/-- The layers of `NTT⁻¹` with `len` in `lens`. -/ -def nttInvLays : List Nat → Prog isa - | [] => .block [] - | len :: lens => .seq (nttLay (bflyInv len) len (-4)) (nttInvLays lens) - -/-- The table `t` to `scratch`, and `rsi` = `f`. -/ -def nttPro (t : Nat → Nat) : List Instr := - [.mov .r9 (.reg .rsi)] ++ storeTab t 256 ++ [.mov .rsi (.reg .rdi), .mov .r8 (.reg .r9)] - -def ntt : Prog isa := - .seq (.block (nttPro zetaTab ++ [.alu .add .r8 (.imm 4)])) (nttLays [128, 64, 32, 16, 8, 4, 2, 1]) - -/-- A coefficient times `8347681`, reduced. -/ -def scaleBody : List Instr := - [.mov32 .rax (.mem (at_ .rsi 0)), .mul .r9] ++ reduce ++ - [.store32 (at_ .rsi 0) .r10, .alu .add .rsi (.imm 4), .alu .sub .rcx (.imm 1)] - -def nttInv : Prog isa := - .seq (.block (nttPro negZetaTab ++ [.alu .add .r8 (.imm (4 * 255))])) - (.seq (nttInvLays [1, 2, 4, 8, 16, 32, 64, 128]) - (.seq (.block [.mov32 .r9 (.imm 8347681)]) - (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block scaleBody) .ne)))) +open VG.Impl.MlKem.X86_64 (xb xmov withMxcsr rcxLoop) + +/-- `ζ^BitRev8(m) · 2³² mod q`. -/ +def zmTab (m : Nat) : Nat := 1753 ^ Spec.MlDsa.bitRev8 m * 2 ^ 32 % 8380417 + +/-- `d ← r + off`. -/ +def leaR (d r : Reg) (off : Nat) : List Instr := [.mov d (.reg r), .alu .add d (.imm (BitVec.ofNat 32 off))] + +/-- The zetas at `[r8]` in the doublewords of `xmm13`, arranged by `pshufd` +with `o`, and its odd doublewords in the even ones of `xmm12`. -/ +def vzeta (o : BitVec 8) : List Instr := + [.movdquLoad .xmm13 (at_ .r8 0), .xop (.pshufd .xmm13 .xmm13 o), .xop (.pshufd .xmm12 .xmm13 0xF5)] + +/-- A layer with `len ≥ 4` and butterflies `bf`: its `128 / len` blocks, the +first with the zeta `k`, the zeta pointer moving by `dz` bytes. -/ +def vlay (bf : List Instr) (len k : Nat) (dz : BitVec 32) : Prog isa := + .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k) ++ + [.mov32 .rax (.imm (BitVec.ofNat 32 (128 / len)))])) <| + .loop (.seq (.block (vzeta 0 ++ [.alu .add .r8 (.imm dz)])) + (.seq (rcxLoop (len / 4) ([.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm1 (at_ .rdx (4 * len))] ++ + bf ++ [.movdquStore (at_ .rdx 0) .xmm0, .movdquStore (at_ .rdx (4 * len)) .xmm3, + .alu .add .rdx (.imm 16)])) + (.block [.alu .add .rdx (.imm (BitVec.ofNat 32 (4 * len))), .alu .sub .rax (.imm 1)]))) .ne + +/-- The layer with `len = 2`, two blocks at a time: the zetas at `[r8]` +arranged by `pshufd` with `o`, the zeta pointer moving by `dz` bytes. -/ +def vlay2 (bf : List Instr) (k : Nat) (o : BitVec 8) (dz : BitVec 32) : Prog isa := + .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k))) <| + rcxLoop 32 ([.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm1 (at_ .rdx 16)] ++ vzeta o ++ + [.alu .add .r8 (.imm dz), xmov .xmm2 .xmm0, xb .punpcklqdq .xmm0 .xmm1, xb .punpckhqdq .xmm2 .xmm1, + xmov .xmm1 .xmm2] ++ bf ++ + [xmov .xmm1 .xmm0, xb .punpcklqdq .xmm0 .xmm3, xb .punpckhqdq .xmm1 .xmm3, + .movdquStore (at_ .rdx 0) .xmm0, .movdquStore (at_ .rdx 16) .xmm1, .alu .add .rdx (.imm 32)]) + +/-- The layer with `len = 1`, four blocks at a time: the zetas at `[r8]` +arranged by `pshufd` with `o`, the zeta pointer moving by `dz` bytes. -/ +def vlay1 (bf : List Instr) (k : Nat) (o : BitVec 8) (dz : BitVec 32) : Prog isa := + .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k))) <| + rcxLoop 32 ([.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm2 (at_ .rdx 16)] ++ vzeta o ++ + [.alu .add .r8 (.imm dz), .xop (.pshufd .xmm0 .xmm0 0xD8), .xop (.pshufd .xmm2 .xmm2 0xD8), + xmov .xmm1 .xmm0, xb .punpcklqdq .xmm0 .xmm2, xb .punpckhqdq .xmm1 .xmm2] ++ bf ++ + [xmov .xmm1 .xmm0, xb .punpckldq .xmm0 .xmm3, xb .punpckhdq .xmm1 .xmm3, + .movdquStore (at_ .rdx 0) .xmm0, .movdquStore (at_ .rdx 16) .xmm1, .alu .add .rdx (.imm 32)]) + +/-- Every coefficient times `8347681 = 256⁻¹ mod q`, reduced. -/ +def vscale : Prog isa := + .seq (.block [.mov .rdx (.reg .rdi), .mov32 .rax (.imm 16382), .xop (.movq .xmm13 .rax), + .xop (.pshufd .xmm13 .xmm13 0), xmov .xmm12 .xmm13]) + (rcxLoop 64 ([.movdquLoad .xmm3 (at_ .rdx 0)] ++ vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ + vcsub .xmm3 .xmm2 ++ [.movdquStore (at_ .rdx 0) .xmm3, .alu .add .rdx (.imm 16)])) + +/-- The table and the constants. -/ +def vpro : List Instr := dwordTab zmTab 256 .rsi ++ vconsts + +def ntt : Prog isa := withMxcsr .rsi 768 <| + .seq (.block vpro) (.seq (vlay vbfly 128 1 4) (.seq (vlay vbfly 64 2 4) (.seq (vlay vbfly 32 4 4) + (.seq (vlay vbfly 16 8 4) (.seq (vlay vbfly 8 16 4) (.seq (vlay vbfly 4 32 4) + (.seq (vlay2 vbfly 64 0x50 8) (vlay1 vbfly 128 0xE4 16)))))))) + +def nttInv : Prog isa := withMxcsr .rsi 768 <| + .seq (.block vpro) (.seq (vlay1 vibfly 252 0x1B (-16)) (.seq (vlay2 vibfly 126 0x05 (-8)) + (.seq (vlay vibfly 4 63 (-4)) (.seq (vlay vibfly 8 31 (-4)) (.seq (vlay vibfly 16 15 (-4)) + (.seq (vlay vibfly 32 7 (-4)) (.seq (vlay vibfly 64 3 (-4)) (.seq (vlay vibfly 128 1 (-4)) + vscale)))))))) end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Vec.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Vec.lean new file mode 100644 index 000000000..1feaab831 --- /dev/null +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Vec.lean @@ -0,0 +1,88 @@ +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Common +import VerifiedGarbage.Impl.MlKem.X86_64.Vec + +/-! +# ML-DSA on x86-64: arithmetic modulo `q` in the doublewords of SSE registers + +The NTT and its inverse compute on four coefficients at a time, as the +doublewords of SSE2 registers, with `q` in the doublewords of `xmm15` and +`-q⁻¹ mod 2³² = 4236238847` in those of `xmm14` (`vconsts`). + +* `vmont d z zo t u`: `d ← d · z · 2⁻³² mod q`, in `[0, 2q)`, for any + doublewords `d` and `z < q` (a Montgomery reduction). `pmuludq` multiplies + the even doublewords of its operands into quadwords, so the even + doublewords of `d` are multiplied by those of `z`, and the odd ones, + moved to the even places of `u` by `pshufd`, by the even doublewords of + `zo`, which hold the odd doublewords of `z` (`pshufd` with `0xF5`). For + each product `P < 2³² · q`, `m = (P mod 2³²) · (-q⁻¹) mod 2³²` makes + `P + m · q` a multiple of `2³²` (with `pmuludq` by `xmm14`, then by + `xmm15`, which use the low doubleword of the product), less than `2⁶⁴`, + and its high doubleword is `(P + m · q) / 2³² < 2q`, congruent to + `P · 2⁻³²` modulo `q`. The quotients of the even doublewords are moved + down to their places by `psrlq`; those of the odd ones are in place, and + the low doublewords of their quadwords are 0, so `por` merges them. +* `vcadd d t`: `d ← d + q` for the doublewords of `d` that are negative + (with `psrad` by 31, a mask), from `(-q, q)` to `[0, q)`; `vcsub d t`: + `d ← d - q`, then `vcadd`, from `[0, 2q)` to `[0, q)`. + +A coefficient `x` is multiplied by `ζ` as `vmont` with `ζ · 2³² mod q`, +which the tables hold. + +`pmuludq` has data-dependent timing on processors with MCDT unless MXCSR is +`0x1FBF` (see `TCB/X86_64/Isa.lean`): the functions run inside ML-KEM's +`withMxcsr`. +-/ + +namespace VG.Impl.MlDsa.X86_64.Arith + +open VG.X86_64 +open VG.Impl.MlKem.X86_64 (xb xmov) + +/-- `q` in the doublewords of `xmm15` and `-q⁻¹ mod 2³² = 4236238847` in +those of `xmm14`, through `rax`. -/ +def vconsts : List Instr := + [.mov32 .rax (.imm 8380417), .xop (.movq .xmm15 .rax), .xop (.pshufd .xmm15 .xmm15 0), + .mov32 .rax (.imm 4236238847), .xop (.movq .xmm14 .rax), .xop (.pshufd .xmm14 .xmm14 0)] + +/-- The Montgomery reductions of the quadword products in `d`, with a +temporary `t`: each quadword becomes `P + m · q`. -/ +def vredc (d t : XReg) : List Instr := + [xmov t d, xb .pmuludq t .xmm14, xb .pmuludq t .xmm15, xb .paddq d t] + +/-- `d ← d · z · 2⁻³² mod q`, in `[0, 2q)`, with the odd doublewords of `z` +in the even doublewords of `zo`, and temporaries `t` and `u`. -/ +def vmont (d z zo t u : XReg) : List Instr := + [.xop (.pshufd u d 0xF5), xb .pmuludq d z, xb .pmuludq u zo] ++ vredc d t ++ + [.xop (.shift .psrlq d 32)] ++ vredc u t ++ [xb .por d u] + +/-- `d ← d + q` for the negative doublewords of `d`, with a temporary `t`. -/ +def vcadd (d t : XReg) : List Instr := + [xmov t d, .xop (.shift .psrad t 31), xb .pand t .xmm15, xb .paddd d t] + +/-- `d ← d mod q` for doublewords in `[0, 2q)`, with a temporary `t`. -/ +def vcsub (d t : XReg) : List Instr := xb .psubd d .xmm15 :: vcadd d t + +/-- The butterflies of Algorithm 41 on the doublewords of `xmm0` (`w[j]`) +and `xmm1` (`w[j + len]`) with the zetas `ζ · 2³² mod q` in `xmm13` (and +its odd doublewords in the even ones of `xmm12`): `xmm0 ← xmm0 + ζ · xmm1` +and `xmm3 ← xmm0 - ζ · xmm1`. -/ +def vbfly : List Instr := + vmont .xmm1 .xmm13 .xmm12 .xmm2 .xmm4 ++ vcsub .xmm1 .xmm2 ++ + (xmov .xmm3 .xmm0 :: xb .paddd .xmm0 .xmm1 :: vcsub .xmm0 .xmm2) ++ + (xb .psubd .xmm3 .xmm1 :: vcadd .xmm3 .xmm2) + +/-- The butterflies of Algorithm 42 on the doublewords of `xmm0` (`w[j]`) +and `xmm1` (`w[j + len]`) with the zetas `ζ · 2³² mod q` in `xmm13` (and +`xmm12`): `xmm0 ← xmm0 + xmm1` and `xmm3 ← ζ · (xmm1 - xmm0)` (Algorithm +42 multiplies `w[j] - w[j + len]` by `-ζ`), from `xmm1 - xmm0 + q`. -/ +def vibfly : List Instr := + (xmov .xmm3 .xmm1 :: xb .psubd .xmm3 .xmm0 :: xb .paddd .xmm3 .xmm15 :: xb .paddd .xmm0 .xmm1 :: + vcsub .xmm0 .xmm2) ++ vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ vcsub .xmm3 .xmm2 + +/-- The `u64`s `t (2i) + 2³² · t (2i + 1)` for `i < n / 2` at `[r + 8i]`, +through `r9`. -/ +def dwordTab (t : Nat → Nat) (n : Nat) (r : Reg) : List Instr := + (List.range (n / 2)).flatMap fun i => + [.movImm64 .r9 (BitVec.ofNat 64 (t (2 * i) + 2 ^ 32 * t (2 * i + 1))), .store (at_ r (8 * i)) .r9] + +end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/Framework/X86_64/Mxcsr.lean b/lean/VerifiedGarbage/Proof/Framework/X86_64/Mxcsr.lean index d6590c716..c276269aa 100644 --- a/lean/VerifiedGarbage/Proof/Framework/X86_64/Mxcsr.lean +++ b/lean/VerifiedGarbage/Proof/Framework/X86_64/Mxcsr.lean @@ -54,6 +54,31 @@ def ctlOk : Prog isa → Bool | .call _ b => ctlOk b | .frame i b j => !loadsMxcsr i && ctlOk b && !loadsMxcsr j +/-- A check that implies `ctlOk` and composes like `Code.allInstrs`: `c` +itself never loads MXCSR, and the functions it calls satisfy `ctlOk`. -/ +def ctlC : Prog isa → Bool + | .block is => is.all fun i => !loadsMxcsr i + | .seq a b => ctlC a && ctlC b + | .ite _ t e => ctlC t && ctlC e + | .loop b _ => ctlC b + | .call _ b => ctlOk b + | .frame i b j => !loadsMxcsr i && ctlC b && !loadsMxcsr j + +theorem ctlOk_of_ctlC {c : Prog isa} (h : ctlC c = true) : ctlOk c = true := by + induction c with + | block _ => exact h + | seq _ _ iha ihb => + simp only [ctlC, Bool.and_eq_true] at h + simp only [ctlOk, iha h.1, ihb h.2, Bool.and_self, Bool.or_true] + | ite _ _ _ iht ihe => + simp only [ctlC, Bool.and_eq_true] at h + simp only [ctlOk, iht h.1, ihe h.2, Bool.and_self] + | loop _ _ ih => exact ih h + | call _ _ _ => exact h + | frame _ _ _ ih => + simp only [ctlC, Bool.and_eq_true] at h + simp only [ctlOk, h.1.1, h.2, ih h.1.2, Bool.and_self] + /-- MXCSR's control bits, which the calling convention preserves. -/ abbrev ctl (v : BitVec 32) : BitVec 10 := v.extractLsb' 6 10 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Ntt.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Ntt.lean index 0b1f7e992..fbd108838 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Ntt.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Ntt.lean @@ -1,12 +1,17 @@ -import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttLoop +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay21 +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Basic +import VerifiedGarbage.Proof.MlKem.X86_64.VMxcsr import VerifiedGarbage.Proof.Framework.X86_64.Abi +import VerifiedGarbage.Proof.Framework.Range /-! # ML-DSA on x86-64: `vg_mldsa_ntt` -Untrusted: everything here is checked by Lean. The butterfly's code does -what `bfly` does (`bfly_spec`), so each layer is `nttLayer` (`lay_ok`), and -the eight layers are `NTT` (`ntt_eq_layers`). `Ntt.LI`, `Ntt.pro_ok` and +Untrusted: everything here is checked by Lean. ML-KEM's `withMxcsr` runs +its code from any MXCSR and keeps what it does (`withMxcsr_ok`); the +prologue leaves the table of zetas in `scratch` and the constants +(`vpro_ok`), each layer is `nttLayer` (`vlay_ok`, `vlay2_ok`, `vlay1_ok`), +and the eight layers are `NTT` (`ntt_eq_layers`). `LI`, `vpro_ok` and `inPlaceSat` serve `NTT⁻¹` too. -/ @@ -14,99 +19,233 @@ namespace VG.Proof.MlDsa.X86_64.Arith open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith open VG.Proof.MlDsa.Arith -open VG.Proof.MlKem.X86_64 (Keep WP.keep writesOnly gprPreserved_of) +open VG.Proof.MlKem.X86_64 (Keep WP.keep writesOnly gprPreserved_of withMxcsr_ok mxR mx_sub xmm_setXmm + GOnly add_ofNat_zero sel) open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs zetas ntt) -namespace Ntt - -/-- Between layers: the polynomial `F` at `fP`, the table `tab` at `zP`, and -entry `k` of the table at `r8`. -/ -structure LI (tab : Nat → Nat) (s₀ : State) (fP zP : Addr) (F : Poly) (k : Nat) (s : State) : Prop where - rsi : s.gpr .rsi = fP - r8 : s.gpr .r8 = coeffAddr zP k - poly : PolyIs s.mem fP F - rd : s.rd = s₀.rd - wr : s.wr = s₀.wr - tab : Tab tab s.mem zP 256 - frame : Frame [pR fP, pR zP] s₀.mem s.mem - keep : Keep [.rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11] s₀ s - -theorem LI.step {tab : Nat → Nat} {s₀ : State} {fP zP : Addr} {F F' : Poly} {k k' : Nat} {s s' : State} - (hI : LI tab s₀ fP zP F k s) (hP : PolyIs s'.mem fP F') (hf : Frame [pR fP] s.mem s'.mem) - (hsi : s'.gpr .rsi = fP) (h8 : s'.gpr .r8 = coeffAddr zP k') - (hk : Keep [.rdi, .r9, .r8, .rcx, .rax, .rdx, .rsi, .rcx, .r10, .r11, .rsi, .rdi, .rsi] s s') - (hd : (pR zP).Disjoint (pR fP)) : LI tab s₀ fP zP F' k' s' := - ⟨hsi, h8, hP, hk.2.1.trans hI.rd, hk.2.2.trans hI.wr, hI.tab.frame hf (by simpa using hd) (by decide), - hI.frame.trans (hf.mono (by simp)), (hI.keep.trans hk).mono (by decide)⟩ - -/-- The chain of zeta indices of the layers `ls` of `NTT`, from `k`. -/ -def Chain : Nat → List Nat → Prop - | _, [] => True - | k, len :: ls => k = 128 / len ∧ Chain (256 / len) ls - -theorem lens_fwd : ∀ len ∈ nttLens, 2 * (128 / len) ≤ 256 ∧ 128 / len + 128 / len = 256 / len := by decide - -theorem zetaTab_of : TabOf zetaTab zetas := fun k _ => zetaNat_eq k - -theorem lays_ok {s₀ : State} {fP zP : Addr} (hw : pR fP ∈ s₀.wr) (hz : pR zP ∈ s₀.rd ++ s₀.wr) - (hd : (pR zP).Disjoint (pR fP)) : - ∀ (ls : List Nat) (F : Poly) (k : Nat) (s : State), (∀ len ∈ ls, len ∈ nttLens) → Chain k ls → - LI zetaTab s₀ fP zP F k s → - WP isa (nttLays ls) s fun s' => ∃ k', LI zetaTab s₀ fP zP (ls.foldl nttLayer F) k' s' - | [], F, k, s, _, _, hI => WP.block_nil ⟨k, hI⟩ - | len :: ls, F, k, s, hls, ⟨hk, hc⟩, hI => by - have hlen := hls len (List.mem_cons_self ..) - obtain ⟨h1, h2⟩ := lens_fwd len hlen - refine WP.seq (WP.mono (lay_ok bfly_spec zetaTab_of hlen 4 (fun c => 128 / len + c) (fun c hc => by omega) - (fun c _ => by rw [show BitVec.signExtend 64 (4 : BitVec 32) = 4 by decide, coeffAddr_succ]; rfl) - F s hI.rsi (by rw [hI.r8, hk]; rfl) hI.poly (by rw [hI.wr]; exact hw) (by rw [hI.rd, hI.wr]; exact hz) hd - hI.tab) fun s' ⟨⟨hP, hf, hsi, h8⟩, hk'⟩ => ?_) - exact lays_ok hw hz hd ls _ (256 / len) s' (fun l hl => hls l (List.mem_cons_of_mem _ hl)) hc - (hI.step hP hf hsi (by rw [h8, h2]) hk' hd) - -/-- The table `tab` to `scratch`, `rsi` = `f` and `r8` at entry `k`. -/ -theorem pro_ok {s₀ : State} {t : Poly → Poly} (hp : (inPlaceK t).pre s₀) (tab : Nat → Nat) (d : BitVec 32) - (k : Nat) (hd : BitVec.signExtend 64 d = BitVec.ofNat 64 (4 * k)) : - WP isa (.block (nttPro tab ++ ([.alu .add .r8 (.imm d)] : List Instr))) s₀ - (LI tab s₀ (s₀.gpr .rdi) (s₀.gpr .rsi) (polyAt s₀.mem (s₀.gpr .rdi)) k) := by - simp only [nttPro, List.append_assoc] - rw [WP.block_append_iff] - refine WP.mono (WP.keep [.r9] (Q := fun s => s.mem = s₀.mem ∧ s.gpr .r9 = s₀.gpr .rsi) (by xrund) (by decide)) - fun s1 ⟨⟨hm1, h9⟩, k1⟩ => ?_ - rw [WP.block_append_iff] - refine WP.mono (storeTab_ok tab (by decide) s1 (by rw [k1.2.2, hp.2.1, h9]; simp)) - fun s2 ⟨ht, hf, k2⟩ => ?_ - have k12 := k1.trans k2 - refine WP.mono (WP.keep [.rsi, .r8] (Q := fun s => s.mem = s2.mem ∧ s.gpr .rsi = s2.gpr .rdi ∧ - s.gpr .r8 = s2.gpr .r9 + BitVec.signExtend 64 d) (by xrund [List.cons_append, List.nil_append]) (by rfl)) - fun s3 ⟨⟨hm3, hsi, h8⟩, k3⟩ => ?_ - have hd' : (pR (s₀.gpr .rdi)).Disjoint (pR (s₀.gpr .rsi)) := hp.2.2.1 - rw [h9] at hf - refine ⟨by rw [hsi, k12.gpr (by decide)], by rw [h8, k2.gpr (by decide), h9, hd], ?_, by rw [k3.2.1, k12.2.1], - by rw [k3.2.2, k12.2.2], by rw [hm3, ← h9]; exact ht, - by rw [hm3, ← hm1]; exact hf.mono (by simp), ((k12.trans k3)).mono (by decide)⟩ - rw [hm3] - exact ⟨reduced_frame (by rw [← hm1]; exact hf) (by simpa using hd') hp.2.2.2.2.2, - polyAt_frame (by rw [← hm1]; exact hf) (by simpa using hd')⟩ - -end Ntt - -theorem chain_fwd : Ntt.Chain 1 nttLens := by - simp only [nttLens, Ntt.Chain]; decide +/-! ## The prologue -/ + +/-- A table of 256 `u32`s `t k`, stored at `sP` (in `r`), two at a time +through `r9`. -/ +theorem dwordTab_ok (t : Nat → Nat) (ht : ∀ k, t k < 2 ^ 32) {r : Reg} (hr : r ≠ .r9) {sP : Addr} {s : State} + (hsi : s.gpr r = sP) (hw : pR sP ∈ s.wr) : + WP isa (.block (dwordTab t 256 r)) s fun s' => Tab t s'.mem sP 256 ∧ + Frame [pR sP] s.mem s'.mem ∧ Keep [.r9] s s' ∧ s'.mxcsr = s.mxcsr ∧ s'.xmm = s.xmm := by + refine WP.mono (wp_range_flatMap (M := isa) (N := 128) (fun i w => + Tab t w.mem sP (2 * i) ∧ Frame [pR sP] s.mem w.mem ∧ + Keep [.r9] s w ∧ w.mxcsr = s.mxcsr ∧ w.xmm = s.xmm) + (fun i w hi ⟨hT, hf, hk, hm, hx⟩ => ?_) 128 (Nat.le_refl _) s + ⟨fun _ h => absurd h (by omega), Frame.refl _ _, Keep.refl _ _, rfl, rfl⟩) + fun w ⟨hT, hf, hk, hm, hx⟩ => ⟨hT, hf, hk, hm, hx⟩ + have hsi' : w.gpr r = sP := by + rw [hk.gpr (by simp only [List.mem_singleton]; exact hr), hsi] + have w0 : InRegions w.wr (sP + BitVec.ofNat 64 (8 * i)) 8 := + ⟨_, by rw [hk.2.2]; exact hw, Offset.contains_base sP (by omega) (by omega)⟩ + have hV : ∀ e < 2, (BitVec.ofNat 64 (t (2 * i) + 2 ^ 32 * t (2 * i + 1))).extractLsb' (32 * e) 32 = + BitVec.ofNat 32 (t (2 * i + e)) := fun e he => by + apply BitVec.eq_of_toNat_eq + have h0 := ht (2 * i) + have h1 := ht (2 * i + 1) + rw [BitVec.extractLsb'_toNat, BitVec.toNat_ofNat, BitVec.toNat_ofNat, Nat.shiftRight_eq_div_pow] + rcases (by omega : e = 0 ∨ e = 1) with rfl | rfl + · simp only [Nat.mul_zero, Nat.pow_zero, Nat.div_one, Nat.add_zero]; omega + · simp only [Nat.mul_one]; omega + vrund [hsi', w0, hr] + generalize BitVec.ofNat 64 (t (2 * i) + 2 ^ 32 * t (2 * i + 1)) = V at hV ⊢ + refine ⟨fun k hk' => ?_, hf.writeW (List.mem_singleton_self _) _ + (Offset.contains_base sP (by omega) (by omega)), + ⟨fun r hr => ?_, hk.2.1, hk.2.2⟩, hm, hx⟩ + · by_cases h : 2 * i ≤ k + · rw [coeffAt_eq, coeffAddr, show sP + BitVec.ofNat 64 (4 * k) = + sP + BitVec.ofNat 64 (8 * i) + BitVec.ofNat 64 (4 * (k - 2 * i)) by + rw [BitVec.add_assoc, ← BitVec.ofNat_add]; exact congrArg _ (congrArg _ (by omega)), + show 32 = 8 * 4 from rfl, readW_writeW_inside _ _ _ (by omega) (by decide), + show 8 * (4 * (k - 2 * i)) = 32 * (k - 2 * i) by omega, hV _ (by omega), + show 2 * i + (k - 2 * i) = k by omega] + · rw [coeffAt_eq, Mem.readW_writeW_sep (Offset.sep sP (by omega) (by omega) (by omega)) (by decide)] + exact hT k (by omega) + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + simp only [RegUpd.gpr_setReg, hr, ite_false] + exact hk.1 r (by simp [hr]) + +theorem vconsts_ok (s : State) : + WP isa (.block vconsts) s fun s' => VConsts s' ∧ Keep [.rax] s s' ∧ s'.mem = s.mem ∧ + s'.mxcsr = s.mxcsr := by + simp only [vconsts] + vrund + refine ⟨⟨?_, ?_⟩, ⟨fun r hr => ?_, rfl, rfl⟩⟩ + · simp only [RegUpd.xmm_setReg, xmm_setXmm, ite_true]; decide + · simp only [RegUpd.xmm_setReg, xmm_setXmm, ite_true, ite_false, reduceCtorEq]; decide + · simp only [List.mem_singleton] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setXmm, hr, ite_false] + +/-- The table of zetas and the constants. -/ +theorem vpro_ok {sP : Addr} {s : State} (hsi : s.gpr .rsi = sP) (hw : pR sP ∈ s.wr) : + WP isa (.block vpro) s fun s' => Tab zmTab s'.mem sP 256 ∧ VConsts s' ∧ + Frame [pR sP] s.mem s'.mem ∧ Keep [.r9, .rax] s s' ∧ s'.mxcsr = s.mxcsr := by + rw [vpro, WP.block_append_iff] + refine WP.mono (dwordTab_ok zmTab (fun k => Nat.lt_trans (zmTab_lt k) (by decide)) (by decide) hsi hw) + fun s1 ⟨hT, hf, k1, x1, _⟩ => WP.mono (vconsts_ok s1) fun s2 ⟨hc, k2, m2, x2⟩ => + ⟨by rw [m2]; exact hT, hc, by rw [m2]; exact hf, (k1.trans k2).mono (by simp), by rw [x2, x1]⟩ + +/-! ## The layers -/ + +/-- Between the layers: the polynomial `F` at `fP`, the table at `sP`, and +the constants. -/ +structure LI (fP sP : Addr) (s₀ : State) (F : Poly) (s : State) : Prop where + P : PolyIs s.mem fP F + T : Tab zmTab s.mem sP 256 + c : VConsts s + keep : Keep [.rax, .rcx, .rdx, .r8] s₀ s + frame : Frame [pR fP] s₀.mem s.mem + +/-- The last layer. -/ +theorem LI.last {fP sP : Addr} {s₀ : State} (hdi : s₀.gpr .rdi = fP) (hsi : s₀.gpr .rsi = sP) + (hwf : pR fP ∈ s₀.wr) (hw : pR sP ∈ s₀.wr) (hd : (pR sP).Disjoint (pR fP)) {l : Prog isa} + {F F' : Poly} + (hl : ∀ s, VConsts s → s.gpr .rdi = fP → s.gpr .rsi = sP → PolyIs s.mem fP F → Tab zmTab s.mem sP 256 → + pR fP ∈ s.wr → pR sP ∈ s.wr → WP isa l s fun s' => PolyIs s'.mem fP F' ∧ BInv fP s s') + {s : State} (hI : LI fP sP s₀ F s) : WP isa l s (LI fP sP s₀ F') := + WP.mono (hl s hI.c (by rw [hI.keep.gpr (by decide), hdi]) (by rw [hI.keep.gpr (by decide), hsi]) hI.P + hI.T (by rw [hI.keep.2.2]; exact hwf) (by rw [hI.keep.2.2]; exact hw)) + fun s' ⟨hS, hb⟩ => ⟨hS, hI.T.frame hb.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) + (by decide), hb.consts, (hI.keep.trans hb.keep).mono (by decide), hI.frame.trans hb.frame⟩ + +/-- A layer, then `c`. -/ +theorem LI.seq {fP sP : Addr} {s₀ : State} (hdi : s₀.gpr .rdi = fP) (hsi : s₀.gpr .rsi = sP) + (hwf : pR fP ∈ s₀.wr) (hw : pR sP ∈ s₀.wr) (hd : (pR sP).Disjoint (pR fP)) {l c : Prog isa} + {F F' : Poly} {Q : State → Prop} + (hl : ∀ s, VConsts s → s.gpr .rdi = fP → s.gpr .rsi = sP → PolyIs s.mem fP F → Tab zmTab s.mem sP 256 → + pR fP ∈ s.wr → pR sP ∈ s.wr → WP isa l s fun s' => PolyIs s'.mem fP F' ∧ BInv fP s s') + (hc : ∀ s, LI fP sP s₀ F' s → WP isa c s Q) {s : State} (hI : LI fP sP s₀ F s) : + WP isa (.seq l c) s Q := + WP.seq (WP.mono (hl s hI.c (by rw [hI.keep.gpr (by decide), hdi]) (by rw [hI.keep.gpr (by decide), hsi]) hI.P + hI.T (by rw [hI.keep.2.2]; exact hwf) (by rw [hI.keep.2.2]; exact hw)) + fun s' ⟨hS, hb⟩ => hc s' ⟨hS, hI.T.frame hb.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) + (by decide), hb.consts, (hI.keep.trans hb.keep).mono (by decide), hI.frame.trans hb.frame⟩) + +theorem step_fwd (p : Addr) (a d : Nat) {dz : BitVec 32} (h : BitVec.signExtend 64 dz = BitVec.ofNat 64 (4 * d)) : + coeffAddr p a + BitVec.signExtend 64 dz = coeffAddr p (a + d) := by + rw [h, coeffAddr_add] + +theorem step_bwd (p : Addr) (a d : Nat) {dz : BitVec 32} + (h : BitVec.ofNat 64 (4 * d) + BitVec.signExtend 64 dz = 0) : + coeffAddr p (a + d) + BitVec.signExtend 64 dz = coeffAddr p a := by + rw [← coeffAddr_add, BitVec.add_assoc, h]; exact BitVec.add_zero _ + +/-- The block of `NTT`. -/ +abbrev fwdBlk : Poly → Nat → Nat → Nat → Nat → Poly := fun f len k st t => blockN bfly f len (zetas k) st t + +theorem nttLayer_eq (F : Poly) (len : Nat) : + nttLayer F len = layF fwdBlk F len (fun c => 128 / len + c) (128 / len) := rfl + +/-- A layer of `NTT` with `len ≥ 4`, whose first zeta is `zetas k`. -/ +theorem fwdLay_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) (len k : Nat) (hlen : len ∈ [4, 8, 16, 32, 64, 128]) (hk : 128 / len = k) + {F : Poly} (s : State) (hc : VConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (vlay vbfly len k 4) s fun s' => PolyIs s'.mem fP (nttLayer F len) ∧ BInv fP s s' := by + rw [nttLayer_eq] + exact vlay_ok vbfly_spec nttBlk_ok hlen 4 (fun c => 128 / len + c) (by rw [hk]; rfl) + (fun c hc => by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hlen + rcases hlen with rfl | rfl | rfl | rfl | rfl | rfl <;> omega) + (fun c _ => step_fwd _ _ 1 (by decide)) hc hdi hsi hS hT hwf hw hd + +theorem fwdLay2_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : VConsts s) (hdi : s.gpr .rdi = fP) + (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) + (hw : pR sP ∈ s.wr) : + WP isa (vlay2 vbfly 64 0x50 8) s fun s' => PolyIs s'.mem fP (nttLayer F 2) ∧ BInv fP s s' := by + have hs : ∀ e < 4, sel 0x50 e = e / 2 := by decide + rw [nttLayer_eq, show 128 / 2 = 64 from rfl] + exact vlay2_ok vbfly_spec nttBlk_ok 64 0x50 8 (fun c => 64 + c) (fun i => 64 + 2 * i) rfl + (fun i _ => by omega) (fun i _ e he => by rw [hs e he]; omega) + (fun i _ => (step_fwd _ _ 2 (by decide)).trans (congrArg _ (by omega))) hc hdi hsi hS hT hwf hw hd + +theorem fwdLay1_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : VConsts s) (hdi : s.gpr .rdi = fP) + (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) + (hw : pR sP ∈ s.wr) : + WP isa (vlay1 vbfly 128 0xE4 16) s fun s' => PolyIs s'.mem fP (nttLayer F 1) ∧ BInv fP s s' := by + have hs : ∀ e < 4, sel 0xE4 e = e := by decide + rw [nttLayer_eq, show 128 / 1 = 128 from rfl] + exact vlay1_ok vbfly_spec nttBlk_ok 128 0xE4 16 (fun c => 128 + c) (fun i => 128 + 4 * i) rfl + (fun i _ => by omega) (fun i _ e he => by rw [hs e he]; omega) + (fun i _ => (step_fwd _ _ 4 (by decide)).trans (congrArg _ (by omega))) hc hdi hsi hS hT hwf hw hd + +/-! ## `vg_mldsa_ntt` -/ + +theorem mx_sub' (sP : Addr) : Region.Sub (mxR sP) (pR sP) := mx_sub sP + +/-- The regions of `scratch` within it, and `f`. -/ +theorem frame_fs {fP sP : Addr} {m m' : Mem} {rs : List Region} (h : Frame rs m m') + (hs : ∀ r ∈ rs, Region.Sub r (pR fP) ∨ Region.Sub r (pR sP)) : Frame [pR fP, pR sP] m m' := + h.sub fun r hr => (hs r hr).elim (fun h => ⟨_, List.mem_cons_self .., h⟩) + fun h => ⟨_, List.mem_cons_of_mem _ (List.mem_singleton_self _), h⟩ + +/-- The code in `withMxcsr`, from its state `s1`: the prologue, then the +layers `l`, which leave `F`, then `NTT⁻¹`'s scaling or nothing. -/ +theorem nttBody_ok {t : Poly → Poly} {s s1 : State} (hs : (inPlaceK t).pre s) {l : Prog isa} {G : Poly} + (k1 : Keep [.rax, .r11] s s1) (f1 : Frame [mxR (s.gpr .rsi)] s.mem s1.mem) + (hl : ∀ s2, LI (s.gpr .rdi) (s.gpr .rsi) s2 (polyAt s.mem (s.gpr .rdi)) s2 → s2.gpr .rdi = s.gpr .rdi → + s2.gpr .rsi = s.gpr .rsi → pR (s.gpr .rdi) ∈ s2.wr → pR (s.gpr .rsi) ∈ s2.wr → + WP isa l s2 fun s3 => LI (s.gpr .rdi) (s.gpr .rsi) s2 G s3) : + WP isa (.seq (.block vpro) l) s1 fun s' => + PolyIs s'.mem (s.gpr .rdi) G ∧ Frame [pR (s.gpr .rdi), pR (s.gpr .rsi)] s.mem s'.mem := by + have hw : pR (s.gpr .rsi) ∈ s.wr := by rw [hs.2.1]; simp + have hwf : pR (s.gpr .rdi) ∈ s.wr := by rw [hs.2.1]; simp + have hd : (pR (s.gpr .rdi)).Disjoint (pR (s.gpr .rsi)) := hs.2.2.1 + have hdi1 : s1.gpr .rdi = s.gpr .rdi := k1.gpr (by decide) + have hsi1 : s1.gpr .rsi = s.gpr .rsi := k1.gpr (by decide) + have hF1 : PolyIs s1.mem (s.gpr .rdi) (polyAt s.mem (s.gpr .rdi)) := + polyIs_frame f1 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd.sub_right (mx_sub' _)) + ⟨hs.2.2.2.2.2, rfl⟩ + refine WP.seq (WP.mono (vpro_ok hsi1 (by rw [k1.2.2]; exact hw)) fun s2 ⟨hT, hc, hf2, k2, _⟩ => ?_) + have hF2 : PolyIs s2.mem (s.gpr .rdi) (polyAt s.mem (s.gpr .rdi)) := + polyIs_frame hf2 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) hF1 + refine WP.mono (hl s2 ⟨hF2, hT, hc, Keep.refl _ _, Frame.refl _ _⟩ + (by rw [k2.gpr (by decide), hdi1]) (by rw [k2.gpr (by decide), hsi1]) + (by rw [k2.2.2, k1.2.2]; exact hwf) (by rw [k2.2.2, k1.2.2]; exact hw)) fun s3 hI => ⟨hI.P, ?_⟩ + refine (frame_fs f1 ?_).trans ((frame_fs hf2 ?_).trans (frame_fs hI.frame ?_)) <;> + intro r hr <;> simp only [List.mem_singleton] at hr <;> subst hr + exacts [.inr (mx_sub' _), .inr fun _ h => h, .inl fun _ h => h] + +/-- `withMxcsr` around the body, and the ABI. -/ +theorem mx_correct {t : Poly → Poly} {l : Prog isa} (s : State) (hs : (inPlaceK t).pre s) + (hk : writesOnly [.rax, .rcx, .rdx, .r8, .r9] (.seq (.block vpro) l) = true) + (hctl : ctlOk (VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 (.seq (.block vpro) l)) = true) + (hk' : writesOnly [.rax, .rcx, .rdx, .r8, .r9, .r11] + (VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 (.seq (.block vpro) l)) = true) + (hl : ∀ s1, Keep [.rax, .r11] s s1 → Frame [mxR (s.gpr .rsi)] s.mem s1.mem → + WP isa (.seq (.block vpro) l) s1 fun s' => PolyIs s'.mem (s.gpr .rdi) (t (polyAt s.mem (s.gpr .rdi))) ∧ + Frame [pR (s.gpr .rdi), pR (s.gpr .rsi)] s.mem s'.mem) : + ∃ tr s', Exec isa (VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 (.seq (.block vpro) l)) s tr s' ∧ + abiPreserved s s' ∧ (inPlaceK t).post s s' := by + have hw : pR (s.gpr .rsi) ∈ s.wr := by rw [hs.2.1]; simp + have hd : (pR (s.gpr .rdi)).Disjoint (pR (s.gpr .rsi)) := hs.2.2.1 + have hW := withMxcsr_ok (c := .seq (.block vpro) l) (by decide) [.rax, .rcx, .rdx, .r8, .r9] (by decide) rfl hw + hk (hl) + obtain ⟨tr, s', he, ⟨s2, ⟨hP, hf⟩, hf', -⟩, hk⟩ := WP.keep [.rax, .rcx, .rdx, .r8, .r9, .r11] hW hk' + refine ⟨tr, s', he, abiPreserved_of_ctl hctl he (gprPreserved_of hk (by decide) + (hf.trans (hf'.sub fun r hr => ⟨_, List.mem_cons_of_mem _ (List.mem_singleton_self _), ?_⟩)) + (by simpa using ⟨hs.2.2.2.1, hs.2.2.2.2.1⟩)), ?_⟩ + · rw [List.mem_singleton.mp hr]; exact mx_sub' _ + · exact polyIs_frame hf' (fun r hr => by + rw [List.mem_singleton.mp hr]; exact hd.sub_right (mx_sub' _)) hP theorem ntt_correct (s : State) (hs : (inPlaceK ntt).pre s) : ∃ t s', Exec isa Impl.MlDsa.X86_64.Arith.ntt s t s' ∧ abiPreserved s s' ∧ (inPlaceK ntt).post s s' := by - have hw : pR (s.gpr .rdi) ∈ s.wr := by rw [hs.2.1]; simp - have hz : pR (s.gpr .rsi) ∈ s.rd ++ s.wr := by rw [hs.1, hs.2.1]; simp - obtain ⟨t, s', he, ⟨k, hI⟩, hk⟩ := WP.keep (c := Impl.MlDsa.X86_64.Arith.ntt) - [.rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11] - (WP.seq (WP.mono (Ntt.pro_ok hs zetaTab 4 1 (by decide)) fun s1 hI => - Ntt.lays_ok hw hz hs.2.2.1.symm nttLens _ 1 s1 (fun _ h => h) chain_fwd hI)) (by decide +kernel) - refine ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he (gprPreserved_of hk (by decide) hI.frame - (by simpa using ⟨hs.2.2.2.1, hs.2.2.2.2.1⟩)), ?_⟩ - show PolyIs _ _ _ - rw [ntt_eq_layers] - exact hI.poly + have hd : (pR (s.gpr .rsi)).Disjoint (pR (s.gpr .rdi)) := hs.2.2.1.symm + refine mx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 => + WP.mono (nttBody_ok hs k1 f1 (G := nttLens.foldl nttLayer (polyAt s.mem (s.gpr .rdi))) + fun s2 hI hdi hsi hwf hw => ?_) fun s' ⟨hP, hf⟩ => ⟨by rw [ntt_eq_layers]; exact hP, hf⟩ + simp only [nttLens, List.foldl_cons, List.foldl_nil] + refine LI.seq hdi hsi hwf hw hd (fwdLay_ok hd 128 1 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (fwdLay_ok hd 64 2 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (fwdLay_ok hd 32 4 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (fwdLay_ok hd 16 8 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (fwdLay_ok hd 8 16 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (fwdLay_ok hd 4 32 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (fwdLay2_ok hd) ?_ hI + exact fun _ hI => LI.last hdi hsi hwf hw hd (fwdLay1_ok hd) hI /-- The pointers and `rsp` are public. -/ theorem inPlace_agree {t : Poly → Poly} (s₁ s₂ : State) (_ : (inPlaceK t).pre s₁) (_ : (inPlaceK t).pre s₂) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttBfly.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttBfly.lean deleted file mode 100644 index d836ba0a2..000000000 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttBfly.lean +++ /dev/null @@ -1,201 +0,0 @@ -import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Ntt -import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Basic -import VerifiedGarbage.Proof.MlDsa.Arith.Ntt - -/-! -# ML-DSA on x86-64: the butterflies of `NTT` and `NTT⁻¹` - -Untrusted: everything here is checked by Lean. What one butterfly's code -stores (`bfly_ok`, `bflyInv_ok`), for any `len`, from the words it reads -and the zeta in `r9`; and that it does what the butterfly of the -specification does (`bfly_spec`, `bflyInv_spec`). --/ - -namespace VG.Proof.MlDsa.X86_64.Arith - -open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith -open VG.Proof.MlDsa.Arith -open VG.Proof.MlKem.X86_64 (Keep WP.keep) -open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs) - -/-! ## `NTT` -/ - -/-- The part of `bfly` after the product is reduced. -/ -def bflyTail (len : Nat) : List Instr := - [.mov32 .rax (.mem (at_ .rsi 0)), .mov32 .rdx (.reg .rax), .alu32 .add .rdx (.imm qImm), - .alu32 .sub .rdx (.reg .r10)] ++ csubQ .rdx .r11 ++ - [.store32 (at_ .rsi (4 * len)) .rdx, .alu32 .add .rax (.reg .r10)] ++ csubQ .rax .r11 ++ - [.store32 (at_ .rsi 0) .rax, .alu .add .rsi (.imm 4), .alu .sub .rcx (.imm 1)] - -theorem bfly_eq (len : Nat) : - Impl.MlDsa.X86_64.Arith.bfly len = - ([.mov32 .rax (.mem (at_ .rsi (4 * len))), .mul .r9] : List Instr) ++ (reduce ++ bflyTail len) := by - simp only [Impl.MlDsa.X86_64.Arith.bfly, bflyTail, List.append_assoc] - -theorem bflyHead_ok (len : Nat) (s : State) - (h : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4) : - WP isa (.block ([.mov32 .rax (.mem (at_ .rsi (4 * len))), .mul .r9] : List Instr)) s fun s' => - (s'.gpr .rax = prodW (s.mem.readW (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 32) (s.gpr .r9) ∧ - s'.mem = s.mem) ∧ Keep [.rax, .rdx] s s' := by - refine WP.keep _ ?_ (by rfl) - xrund [h] - -theorem bflyTail_ok (len : Nat) (s : State) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) - (w0 : InRegions s.wr (s.gpr .rsi) 4) (w1 : InRegions s.wr (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4) : - WP isa (.block (bflyTail len)) s fun s' => - (s'.mem = (s.mem.writeW (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) - (csubD (s.mem.readW (s.gpr .rsi) 32 + qImm - BitVec.setWidth 32 (s.gpr .r10)))).writeW (s.gpr .rsi) - (csubD (s.mem.readW (s.gpr .rsi) 32 + BitVec.setWidth 32 (s.gpr .r10))) ∧ - s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rax, .rdx, .rsi, .rcx, .r11] s s' := by - refine WP.keep _ ?_ (by rfl) - unfold bflyTail csubQ - xrund [h0, w0, w1, List.cons_append, List.nil_append, csubD] - -theorem bfly_ok (len : Nat) (s : State) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) - (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4) - (w0 : InRegions s.wr (s.gpr .rsi) 4) (w1 : InRegions s.wr (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4) : - WP isa (.block (Impl.MlDsa.X86_64.Arith.bfly len)) s fun s' => - (s'.mem = (s.mem.writeW (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) - (csubD (s.mem.readW (s.gpr .rsi) 32 + qImm - BitVec.setWidth 32 - (redD (prodW (s.mem.readW (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 32) (s.gpr .r9)))))).writeW - (s.gpr .rsi) (csubD (s.mem.readW (s.gpr .rsi) 32 + BitVec.setWidth 32 - (redD (prodW (s.mem.readW (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 32) (s.gpr .r9))))) ∧ - s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rax, .rdx, .rsi, .rcx, .r10, .r11] s s' := by - rw [bfly_eq, WP.block_append_iff] - refine WP.mono (bflyHead_ok len s h1) fun s1 ⟨⟨ha, hm1⟩, k1⟩ => ?_ - rw [WP.block_append_iff] - refine WP.mono (reduce_ok s1) fun s2 ⟨⟨hr, hm2⟩, k2⟩ => ?_ - have k12 := k1.trans k2 - have hsi : s2.gpr .rsi = s.gpr .rsi := k12.gpr (by decide) - have hrr : s2.rd ++ s2.wr = s.rd ++ s.wr := by rw [k12.2.1, k12.2.2] - refine WP.mono (bflyTail_ok len s2 (by rw [hrr, hsi]; exact h0) (by rw [k12.2.2, hsi]; exact w0) - (by rw [k12.2.2, hsi]; exact w1)) fun s3 ⟨⟨hm3, h3si, h3cx, h3z⟩, k3⟩ => ⟨?_, (k12.trans k3).mono (by decide)⟩ - have hcx : s2.gpr .rcx = s.gpr .rcx := k12.gpr (by decide) - rw [hm3, h3si, h3cx, h3z, hcx, hsi, hr, hm2, hm1, ha] - exact ⟨rfl, rfl, rfl, rfl⟩ - -/-! ## `NTT⁻¹` -/ - -/-- The part of `bflyInv` up to the product. -/ -def bflyInvHead (len : Nat) : List Instr := - [.mov32 .rax (.mem (at_ .rsi 0)), .mov32 .r10 (.mem (at_ .rsi (4 * len))), .mov32 .rdx (.reg .rax), - .alu32 .add .rdx (.reg .r10)] ++ csubQ .rdx .r11 ++ - [.store32 (at_ .rsi 0) .rdx, .alu32 .add .rax (.imm qImm), .alu32 .sub .rax (.reg .r10)] ++ - csubQ .rax .r11 ++ [.mul .r9] - -theorem bflyInv_eq (len : Nat) : - Impl.MlDsa.X86_64.Arith.bflyInv len = bflyInvHead len ++ (reduce ++ ([.store32 (at_ .rsi (4 * len)) .r10, - .alu .add .rsi (.imm 4), .alu .sub .rcx (.imm 1)] : List Instr)) := by - simp only [Impl.MlDsa.X86_64.Arith.bflyInv, bflyInvHead, List.append_assoc] - -theorem bflyInvHead_ok (len : Nat) (s : State) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) - (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4) - (w0 : InRegions s.wr (s.gpr .rsi) 4) : - WP isa (.block (bflyInvHead len)) s fun s' => - (s'.mem = s.mem.writeW (s.gpr .rsi) (csubD (s.mem.readW (s.gpr .rsi) 32 + - s.mem.readW (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 32)) ∧ - s'.gpr .rax = prodW (csubD (s.mem.readW (s.gpr .rsi) 32 + qImm - - s.mem.readW (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 32)) (s.gpr .r9)) ∧ - Keep [.rax, .rdx, .r10, .r11] s s' := by - refine WP.keep _ ?_ (by rfl) - unfold bflyInvHead csubQ - xrund [h0, h1, w0, List.cons_append, List.nil_append, csubD] - -theorem bflyInv_ok (len : Nat) (s : State) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) - (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4) - (w0 : InRegions s.wr (s.gpr .rsi) 4) (w1 : InRegions s.wr (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4) : - WP isa (.block (Impl.MlDsa.X86_64.Arith.bflyInv len)) s fun s' => - (s'.mem = (s.mem.writeW (s.gpr .rsi) (csubD (s.mem.readW (s.gpr .rsi) 32 + - s.mem.readW (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 32))).writeW - (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) (BitVec.setWidth 32 (redD (prodW - (csubD (s.mem.readW (s.gpr .rsi) 32 + qImm - s.mem.readW (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 32)) - (s.gpr .r9)))) ∧ - s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rax, .rdx, .rsi, .rcx, .r10, .r11] s s' := by - rw [bflyInv_eq, WP.block_append_iff] - refine WP.mono (bflyInvHead_ok len s h0 h1 w0) fun s1 ⟨⟨hm1, ha⟩, k1⟩ => ?_ - rw [WP.block_append_iff] - refine WP.mono (reduce_ok s1) fun s2 ⟨⟨hr, hm2⟩, k2⟩ => ?_ - have k12 := k1.trans k2 - have hsi : s2.gpr .rsi = s.gpr .rsi := k12.gpr (by decide) - have hcx : s2.gpr .rcx = s.gpr .rcx := k12.gpr (by decide) - refine WP.mono (WP.keep [.rsi, .rcx] (Q := fun s' => s'.mem = s2.mem.writeW (s2.gpr .rsi + BitVec.ofNat 64 (4 * len)) - (BitVec.setWidth 32 (s2.gpr .r10)) ∧ s'.gpr .rsi = s2.gpr .rsi + 4 ∧ s'.gpr .rcx = s2.gpr .rcx - 1 ∧ - s'.zf = some (s2.gpr .rcx - 1 == 0)) (by - xrund [show InRegions s2.wr (s2.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4 by rw [k12.2.2, hsi]; exact w1]) - (by rfl)) fun s3 ⟨⟨hm3, h3si, h3cx, h3z⟩, k3⟩ => ⟨?_, (k12.trans k3).mono (by decide)⟩ - rw [hm3, h3si, h3cx, h3z, hcx, hsi, hr, hm2, hm1, ha] - exact ⟨rfl, rfl, rfl, rfl⟩ - -/-! ## What they do to a stored polynomial -/ - -/-- The code `code len` of a butterfly does what `op` does. -/ -def BflyOk (code : Nat → List Instr) (op : Poly → Nat → Nat → Zq → Poly) : Prop := - ∀ (fP : Addr) (len j : Nat), 0 < len → j + len < 256 → ∀ (z : Zq) (F : Poly) (s : State), - s.gpr .rsi = coeffAddr fP j → s.gpr .r9 = BitVec.ofNat 64 z.val → PolyIs s.mem fP F → pR fP ∈ s.wr → - WP isa (.block (code len)) s fun s' => (PolyIs s'.mem fP (op F j len z) ∧ Frame [pR fP] s.mem s'.mem ∧ - s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ - Keep [.rax, .rdx, .rsi, .rcx, .r10, .r11] s s' - -/-- The words a butterfly reads. -/ -theorem bfly_regions {fP : Addr} {len j : Nat} (hj : j + len < 256) {s : State} - (hsi : s.gpr .rsi = coeffAddr fP j) (hw : pR fP ∈ s.wr) : - InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4 ∧ InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4 ∧ - InRegions s.wr (s.gpr .rsi) 4 ∧ InRegions s.wr (s.gpr .rsi + BitVec.ofNat 64 (4 * len)) 4 := by - rw [hsi, coeffAddr_add] - exact ⟨⟨_, List.mem_append_right _ hw, coeff_contains _ (show j < 256 by omega)⟩, - ⟨_, List.mem_append_right _ hw, coeff_contains _ hj⟩, ⟨_, hw, coeff_contains _ (show j < 256 by omega)⟩, - ⟨_, hw, coeff_contains _ hj⟩⟩ - -/-- The two writes of a butterfly are within the polynomial. -/ -theorem bfly_frame {fP : Addr} {len j : Nat} (hj : j + len < 256) (m : Mem) (a b : BitVec 32) : - Frame [pR fP] m ((m.writeW (coeffAddr fP (j + len)) a).writeW (coeffAddr fP j) b) ∧ - Frame [pR fP] m ((m.writeW (coeffAddr fP j) a).writeW (coeffAddr fP (j + len)) b) := - ⟨(Frame.refl _ _ |>.writeW (List.mem_singleton_self _) _ (coeff_contains _ hj)).writeW - (List.mem_singleton_self _) _ (coeff_contains _ (show j < 256 by omega)), - (Frame.refl _ _ |>.writeW (List.mem_singleton_self _) _ (coeff_contains _ (show j < 256 by omega))).writeW - (List.mem_singleton_self _) _ (coeff_contains _ hj)⟩ - -theorem bfly_spec : BflyOk Impl.MlDsa.X86_64.Arith.bfly Arith.bfly := by - intro fP len j hlen hj z F s hsi h9 hF hw - obtain ⟨r0, r1, w0, w1⟩ := bfly_regions hj hsi hw - refine WP.mono (bfly_ok len s r0 r1 w0 w1) fun s' ⟨⟨hm, hsi', hcx, hz⟩, hk⟩ => ⟨⟨?_, ?_, hsi', hcx, hz⟩, hk⟩ - · rw [hm, h9, hsi, coeffAddr_add, ← coeffAt_eq, ← coeffAt_eq] - have hj' : j < 256 := by omega - have ha := polyIs_toNat hF hj' - have hT := redD_prodW (z := z) (polyIs_toNat hF hj) - show PolyIs _ _ ((F.set! (j + len) (F[j]! - z * F[j + len]!)).set! j - ((F.set! (j + len) (F[j]! - z * F[j + len]!))[j]! + z * F[j + len]!)) - rw [getElem!_set!_ne _ hj' (by omega)] - exact polyIs_writeW (polyIs_writeW hF hj _ (csubD_sub_val ha hT)) hj' _ (csubD_add_val ha hT) - · rw [hm, hsi, coeffAddr_add] - exact (bfly_frame hj _ _ _).1 - -theorem bflyInv_spec : BflyOk Impl.MlDsa.X86_64.Arith.bflyInv Arith.bflyInv := by - intro fP len j hlen hj z F s hsi h9 hF hw - obtain ⟨r0, r1, w0, w1⟩ := bfly_regions hj hsi hw - refine WP.mono (bflyInv_ok len s r0 r1 w0 w1) fun s' ⟨⟨hm, hsi', hcx, hz⟩, hk⟩ => ⟨⟨?_, ?_, hsi', hcx, hz⟩, hk⟩ - · rw [hm, h9, hsi, coeffAddr_add, ← coeffAt_eq, ← coeffAt_eq] - have hj' : j < 256 := by omega - have ha := polyIs_toNat hF hj' - have hu := polyIs_toNat hF hj - have hne : j ≠ j + len := by omega - show PolyIs _ _ (((F.set! j (F[j]! + F[j + len]!)).set! (j + len) - (F[j]! - (F.set! j (F[j]! + F[j + len]!))[j + len]!)).set! (j + len) - (z * ((F.set! j (F[j]! + F[j + len]!)).set! (j + len) - (F[j]! - (F.set! j (F[j]! + F[j + len]!))[j + len]!))[j + len]!)) - rw [getElem!_set!_ne _ hj hne, getElem!_set!_self _ hj] - have e : ∀ (G : Poly) (x y : Zq), (G.set! (j + len) x).set! (j + len) y = G.set! (j + len) y := fun G x y => - ext_getElem! fun i hi => by - by_cases h : i = j + len - · subst h; rw [getElem!_set!_self _ hi, getElem!_set!_self _ hi] - · rw [getElem!_set!_ne _ hi (Ne.symm h), getElem!_set!_ne _ hi (Ne.symm h), getElem!_set!_ne _ hi (Ne.symm h)] - rw [e] - exact polyIs_writeW (polyIs_writeW hF hj' _ (csubD_add_val ha hu)) hj _ - (redD_prodW (csubD_sub_val ha hu)) - · rw [hm, hsi, coeffAddr_add] - exact (bfly_frame hj _ _ _).2 - -end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttInv.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttInv.lean index bed819100..f2597066f 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttInv.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttInv.lean @@ -3,163 +3,200 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Ntt /-! # ML-DSA on x86-64: `vg_mldsa_inv_ntt` -Untrusted: everything here is checked by Lean. The butterfly's code does -what `bflyInv` does (`bflyInv_spec`), with the negated zetas of the table -(`negZetaTab_of`), so each layer is `nttInvLayer`, the eight layers are -those of `NTT⁻¹` (`nttInv_eq_layers`), and the last loop multiplies each -coefficient by `8347681 = 256⁻¹ mod q`. +Untrusted: everything here is checked by Lean. As `vg_mldsa_ntt` +(`Ntt.lean`): each layer is `nttInvLayer` (with `vibfly`, which multiplies +by `ζ` the difference the other way round: `nttInvBlk_ok`), the eight +layers are those of `NTT⁻¹` (`nttInv_eq_layers`), and the last pass +multiplies each coefficient by `8347681 = 256⁻¹ mod q` (`vscale_ok`). -/ namespace VG.Proof.MlDsa.X86_64.Arith open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith open VG.Proof.MlDsa.Arith -open VG.Proof.MlKem.X86_64 (Keep WP.keep writesOnly gprPreserved_of wp_counted ifp ifn) +open VG.Proof.MlKem.X86_64 (Keep XOnly xmm_setXmm ifp ifn sel GOnly wp_rcxLoop add_ofNat_zero) +open VG.Impl.MlKem.X86_64 (xb xmov) open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs zetas nttInv) -namespace NttInv - -open Ntt (LI) - -/-- The chain of zeta indices of the layers `ls` of `NTT⁻¹`, from `k`. -/ -def Chain : Nat → List Nat → Prop - | _, [] => True - | k, len :: ls => k = 256 / len - 1 ∧ Chain (256 / len - 1 - 128 / len) ls - -theorem lens_inv : ∀ len ∈ nttLens, 128 / len + 1 ≤ 256 / len ∧ 256 / len ≤ 256 := by decide - -theorem sx_m4 : BitVec.signExtend 64 (-4 : BitVec 32) = -4 := by decide - -theorem step_down (zP : Addr) (m : Nat) : - coeffAddr zP (m + 1) + BitVec.signExtend 64 (-4 : BitVec 32) = coeffAddr zP m := by - rw [sx_m4, ← coeffAddr_succ, BitVec.add_assoc, show (4 : BitVec 64) + -4 = 0#64 by decide, BitVec.add_zero] - -theorem negZetaTab_of : TabOf negZetaTab fun k => -zetas k := fun k _ => negZetaNat_eq k - -theorem lays_ok {s₀ : State} {fP zP : Addr} (hw : pR fP ∈ s₀.wr) (hz : pR zP ∈ s₀.rd ++ s₀.wr) - (hd : (pR zP).Disjoint (pR fP)) : - ∀ (ls : List Nat) (F : Poly) (k : Nat) (s : State), (∀ len ∈ ls, len ∈ nttLens) → Chain k ls → - LI negZetaTab s₀ fP zP F k s → - WP isa (nttInvLays ls) s fun s' => ∃ k', LI negZetaTab s₀ fP zP (ls.foldl nttInvLayer F) k' s' - | [], F, k, s, _, _, hI => WP.block_nil ⟨k, hI⟩ - | len :: ls, F, k, s, hls, ⟨hk, hc⟩, hI => by - have hlen := hls len (List.mem_cons_self ..) - obtain ⟨h1, h2⟩ := lens_inv len hlen - refine WP.seq (WP.mono (lay_ok bflyInv_spec negZetaTab_of hlen (-4) (fun c => 256 / len - 1 - c) - (fun c hc => by omega) - (fun c hc => by - rw [show 256 / len - 1 - c = (256 / len - 1 - (c + 1)) + 1 by omega] - exact step_down zP _) - F s hI.rsi (by rw [hI.r8, hk]; rfl) hI.poly (by rw [hI.wr]; exact hw) (by rw [hI.rd, hI.wr]; exact hz) hd - hI.tab) fun s' ⟨⟨hP, hf, hsi, h8⟩, hk'⟩ => ?_) - exact lays_ok hw hz hd ls _ _ s' (fun l hl => hls l (List.mem_cons_of_mem _ hl)) hc - (hI.step hP hf hsi h8 hk' hd) - -theorem chain_inv : Chain 255 nttInvLens := by - simp only [nttInvLens, Chain]; decide - -/-! ## The multiplication by 8347681 -/ - -theorem scaleHead_ok (s : State) (h : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) : - WP isa (.block ([.mov32 .rax (.mem (at_ .rsi 0)), .mul .r9] : List Instr)) s fun s' => - (s'.gpr .rax = prodW (s.mem.readW (s.gpr .rsi) 32) (s.gpr .r9) ∧ s'.mem = s.mem) ∧ - Keep [.rax, .rdx] s s' := by - refine WP.keep _ ?_ (by decide) - xrund [h] - -theorem scaleBody_ok (s : State) (h : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) (w : InRegions s.wr (s.gpr .rsi) 4) : - WP isa (.block scaleBody) s fun s' => - (s'.mem = s.mem.writeW (s.gpr .rsi) (BitVec.setWidth 32 (redD (prodW (s.mem.readW (s.gpr .rsi) 32) - (s.gpr .r9)))) ∧ s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rax, .rdx, .rsi, .rcx, .r10, .r11] s s' := by - unfold scaleBody - simp only [List.append_assoc] +/-! ## The layers -/ + +/-- The block of `NTT⁻¹`. -/ +abbrev invBlk : Poly → Nat → Nat → Nat → Nat → Poly := fun f len k st t => blockN bflyInv f len (-zetas k) st t + +theorem nttInvLayer_eq (F : Poly) (len : Nat) : + nttInvLayer F len = layF invBlk F len (fun c => 256 / len - 1 - c) (128 / len) := rfl + +/-- A layer of `NTT⁻¹` with `len ≥ 4`, whose first zeta is `zetas k`. -/ +theorem invLay_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) (len k : Nat) + (hlen : len ∈ [4, 8, 16, 32, 64, 128]) (hk : 256 / len - 1 = k) + {F : Poly} (s : State) (hc : VConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (vlay vibfly len k (-4)) s fun s' => PolyIs s'.mem fP (nttInvLayer F len) ∧ BInv fP s s' := by + have hl : 128 / len ≥ 1 ∧ 256 / len = 2 * (128 / len) ∧ 256 / len ≤ 64 := by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hlen + rcases hlen with rfl | rfl | rfl | rfl | rfl | rfl <;> decide + rw [nttInvLayer_eq] + exact vlay_ok vibfly_spec nttInvBlk_ok hlen (-4) (fun c => 256 / len - 1 - c) (by rw [hk]; rfl) + (fun c _ => by omega) + (fun c hc' => (congrArg (· + _) (congrArg (coeffAddr sP) (show 256 / len - 1 - c = + 256 / len - 1 - (c + 1) + 1 by omega))).trans (step_bwd _ _ 1 (by decide))) + hc hdi hsi hS hT hwf hw hd + +theorem invLay2_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : VConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (vlay2 vibfly 126 0x05 (-8)) s fun s' => PolyIs s'.mem fP (nttInvLayer F 2) ∧ BInv fP s s' := by + have hs : ∀ e < 4, sel 0x05 e = 1 - e / 2 := by decide + rw [nttInvLayer_eq, show 256 / 2 - 1 = 127 from rfl, show 128 / 2 = 64 from rfl] + exact vlay2_ok vibfly_spec nttInvBlk_ok 126 0x05 (-8) (fun c => 127 - c) (fun i => 126 - 2 * i) rfl + (fun i _ => by omega) (fun i hi e he => by rw [hs e he]; omega) + (fun i hi => (congrArg (· + _) (congrArg (coeffAddr sP) (show 126 - 2 * i = 126 - 2 * (i + 1) + 2 by + omega))).trans (step_bwd _ _ 2 (by decide))) hc hdi hsi hS hT hwf hw hd + +theorem invLay1_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : VConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (vlay1 vibfly 252 0x1B (-16)) s fun s' => PolyIs s'.mem fP (nttInvLayer F 1) ∧ BInv fP s s' := by + have hs : ∀ e < 4, sel 0x1B e = 3 - e := by decide + rw [nttInvLayer_eq, show 256 / 1 - 1 = 255 from rfl, show 128 / 1 = 128 from rfl] + exact vlay1_ok vibfly_spec nttInvBlk_ok 252 0x1B (-16) (fun c => 255 - c) (fun i => 252 - 4 * i) rfl + (fun i _ => by omega) (fun i hi e he => by rw [hs e he]; omega) + (fun i hi => (congrArg (· + _) (congrArg (coeffAddr sP) (show 252 - 4 * i = 252 - 4 * (i + 1) + 4 by + omega))).trans (step_bwd _ _ 4 (by decide))) hc hdi hsi hS hT hwf hw hd + +/-! ## The multiplication by `256⁻¹` -/ + +/-- The coefficients of `G` before `4i` multiplied by `8347681`. -/ +def Scaled (m : Mem) (fP : Addr) (G : Poly) (i : Nat) : Prop := + ∀ k < 256, (coeffAt m fP k).toNat = (if k < 4 * i then G[k]! * 8347681 else G[k]!).val + +/-- `8347681 · 2³² mod q` in the doublewords of `xmm13` and `xmm12`. -/ +theorem scale_zlanes (x : BitVec 128) (hx : x = shufDwords ((0 : BitVec 64) ++ BitVec.setWidth 64 16382#32) 0) : + ZLanes x (fun _ => 8347681) ∧ ZOdd x x := by + subst hx + refine ⟨fun i hi => ?_, fun j hj => ?_⟩ + · rcases cases4 hi with rfl | rfl | rfl | rfl <;> decide + · rcases (by omega : j = 0 ∨ j = 1) with rfl | rfl <;> decide + +/-- The body of the loop of `vscale`. -/ +abbrev sbody : List Instr := + [.movdquLoad .xmm3 (at_ .rdx 0)] ++ vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ + vcsub .xmm3 .xmm2 ++ [.movdquStore (at_ .rdx 0) .xmm3, .alu .add .rdx (.imm 16)] ++ + [.alu .sub .rcx (.imm 1)] + +/-- The product of the doublewords of `xmm3` by the constant, reduced. -/ +theorem vmul3_ok {s : State} (hc : VConsts s) : + WP isa (.block (vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ vcsub .xmm3 .xmm2)) s fun s' => + s'.xmm .xmm3 = csubV (montV (s.xmm .xmm3) (s.xmm .xmm13) (s.xmm .xmm12)) ∧ + XOnly [.xmm3, .xmm2, .xmm4] s s' := by + simp only [vmont, vredc, vcsub, vcadd, xmov, xb, List.cons_append, List.nil_append] + vrun [eval_movdqa] + rw [hc.q, hc.qinv] + exact ⟨rfl, by xonly⟩ + +theorem vscale_step {fP : Addr} {G : Poly} {i : Nat} (hi : i < 64) {s : State} (hc : VConsts s) + (hz : ZLanes (s.xmm .xmm13) (fun _ => 8347681)) (ho : ZOdd (s.xmm .xmm13) (s.xmm .xmm12)) + (hdx : s.gpr .rdx = coeffAddr fP (4 * i)) (hS : Scaled s.mem fP G i) (hw : pR fP ∈ s.wr) : + WP isa (.block sbody) s fun s' => + Scaled s'.mem fP G (i + 1) ∧ s'.gpr .rdx = coeffAddr fP (4 * (i + 1)) ∧ + Frame [pR fP] s.mem s'.mem ∧ VConsts s' ∧ s'.xmm .xmm13 = s.xmm .xmm13 ∧ + s'.xmm .xmm12 = s.xmm .xmm12 ∧ Keep [.rdx, .rcx] s s' ∧ + s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) ∧ s'.mxcsr = s.mxcsr := by + have j0 : 4 * i + 4 ≤ 256 := by omega + have r0 : InRegions (s.rd ++ s.wr) (coeffAddr fP (4 * i)) 16 := f_in (List.mem_append_right _ hw) j0 + have w0 := f_in hw j0 + have hx : DLanes (s.mem.readW (coeffAddr fP (4 * i)) 128) (fun e => G[4 * i + e]!) := fun e he => by + rw [dword_readW _ _ he, coeffAddr_add, ← coeffAt_eq, hS _ (by omega), ifn (by omega)] + rw [show sbody = [.movdquLoad .xmm3 (at_ .rdx 0)] ++ ((vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ + vcsub .xmm3 .xmm2) ++ [.movdquStore (at_ .rdx 0) .xmm3, .alu .add .rdx (.imm 16), .alu .sub .rcx (.imm 1)]) + from rfl, WP.block_append_iff] + vrund [hdx, r0] rw [WP.block_append_iff] - refine WP.mono (scaleHead_ok s h) fun s1 ⟨⟨ha, hm1⟩, k1⟩ => ?_ - rw [WP.block_append_iff] - refine WP.mono (reduce_ok s1) fun s2 ⟨⟨hr, hm2⟩, k2⟩ => ?_ - have k12 := k1.trans k2 - have hsi : s2.gpr .rsi = s.gpr .rsi := k12.gpr (by decide) - have hcx : s2.gpr .rcx = s.gpr .rcx := k12.gpr (by decide) - refine WP.mono (WP.keep [.rsi, .rcx] (Q := fun s' => s'.mem = s2.mem.writeW (s2.gpr .rsi) - (BitVec.setWidth 32 (s2.gpr .r10)) ∧ s'.gpr .rsi = s2.gpr .rsi + 4 ∧ s'.gpr .rcx = s2.gpr .rcx - 1 ∧ - s'.zf = some (s2.gpr .rcx - 1 == 0)) (by - xrund [show InRegions s2.wr (s2.gpr .rsi) 4 by rw [k12.2.2, hsi]; exact w]) - (by decide)) fun s3 ⟨⟨hm3, h3si, h3cx, h3z⟩, k3⟩ => ⟨?_, (k12.trans k3).mono (by decide)⟩ - rw [hm3, h3si, h3cx, h3z, hcx, hsi, hr, hm2, hm1, ha] - exact ⟨rfl, rfl, rfl, rfl⟩ - -/-- After `i` coefficients of `G` multiplied by 8347681, from the state `sL`. -/ -structure SInv (sL : State) (fP : Addr) (G : Poly) (i : Nat) (s : State) : Prop where - rsi : s.gpr .rsi = coeffAddr fP i - r9 : s.gpr .r9 = BitVec.ofNat 64 (8347681 : Zq).val - rd : s.rd = sL.rd - wr : s.wr = sL.wr - frame : Frame [pR fP] sL.mem s.mem - coeff : ∀ k < 256, (coeffAt s.mem fP k).toNat = if k < i then (G[k]! * 8347681).val else (G[k]!).val - keep : Keep [.r9, .rcx, .rax, .rdx, .rsi, .rcx, .r10, .r11] sL s - -theorem scale_step {sL : State} {fP : Addr} {G : Poly} (hw : pR fP ∈ sL.wr) {i : Nat} (hi : i < 256) - {s : State} (hI : SInv sL fP G i s) : - WP isa (.block scaleBody) s fun s' => SInv sL fP G (i + 1) s' ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0) := by - have hw' : pR fP ∈ s.wr := by rw [hI.wr]; exact hw - refine WP.mono (scaleBody_ok s (by rw [hI.rsi]; exact ⟨_, List.mem_append_right _ hw', coeff_contains _ hi⟩) - (by rw [hI.rsi]; exact ⟨_, hw', coeff_contains _ hi⟩)) fun s' ⟨⟨hm, hsi, hcx, hz⟩, hk⟩ => ⟨?_, hcx, hz⟩ - have hv : (BitVec.setWidth 32 (redD (prodW (coeffAt s.mem fP i) (s.gpr .r9)))).toNat = - (G[i]! * 8347681).val := by - rw [hI.r9, redD_prodW (x := G[i]!) (by rw [hI.coeff i hi, ifn (Nat.lt_irrefl i)]), val_mul, val_mul, - Nat.mul_comm] - rw [hI.rsi, ← coeffAt_eq] at hm - refine ⟨by rw [hsi, hI.rsi, coeffAddr_succ], by rw [hk.gpr (by decide), hI.r9], hk.2.1.trans hI.rd, - hk.2.2.trans hI.wr, by rw [hm]; exact hI.frame.writeW (List.mem_singleton_self _) _ (coeff_contains _ hi), - fun k hk' => ?_, (hI.keep.trans hk).mono (by decide)⟩ - rw [hm, coeffAt_writeW _ _ hk' hi] - by_cases e : i = k - · subst e; rw [ifp rfl, ifp (Nat.lt_succ_self _)]; exact hv - · rw [ifn e, hI.coeff k hk'] - by_cases h : k < i - · rw [ifp h, ifp (by omega)] - · rw [ifn h, ifn (by omega)] - -/-- The multiplication of every coefficient by 8347681. -/ -theorem scale_ok {fP : Addr} {G : Poly} (sL : State) (hsi : sL.gpr .rsi = fP) (hG : PolyIs sL.mem fP G) - (hw : pR fP ∈ sL.wr) : - WP isa (.seq (.block [.mov32 .r9 (.imm 8347681)]) - (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block scaleBody) .ne))) sL fun s' => - PolyIs s'.mem fP (G.map (· * 8347681)) ∧ Frame [pR fP] sL.mem s'.mem ∧ - Keep [.r9, .rcx, .rax, .rdx, .rsi, .rcx, .r10, .r11] sL s' := by - refine WP.seq (WP.mono (WP.keep [.r9] - (Q := fun s => s.mem = sL.mem ∧ s.gpr .r9 = BitVec.ofNat 64 (8347681 : Zq).val) - (by xrund) (by decide)) fun s3 ⟨⟨hm3, h93⟩, k3⟩ => ?_) - refine WP.mono (wp_counted (s₀ := s3) (N := 256) (v := 256) rfl (by decide) (SInv sL fP G) - (fun s4 hm4 hk4 => ⟨by rw [hk4.gpr (by decide), k3.gpr (by decide), hsi]; simp, - by rw [hk4.gpr (by decide), h93], hk4.2.1.trans k3.2.1, hk4.2.2.trans k3.2.2, - by rw [hm4, hm3]; exact Frame.refl _ _, - fun k hk => by rw [ifn (Nat.not_lt_zero _), hm4, hm3]; exact polyIs_toNat hG hk, - (k3.trans hk4).mono (by decide)⟩) - fun i hi s hI => scale_step hw hi hI) fun s' hI => ⟨?_, hI.frame, hI.keep⟩ - refine polyIs_of_toNat fun k hk => ?_ - rw [hI.coeff k hk, ifp hk, map_mul_get _ _ hk] - -end NttInv + refine WP.mono (vmul3_ok (hc.setXmm (by decide) (by decide) _)) fun s2 ⟨h3, o2⟩ => ?_ + have c2 := xonly_vconsts o2 (hc.setXmm (by decide) (by decide) _) (by decide) (by decide) + have g2 : s2.gpr = s.gpr := o2.gpr + have m2 : s2.mem = s.mem := o2.mem + have e2 : s2.rd = s.rd ∧ s2.wr = s.wr := ⟨o2.rd, o2.wr⟩ + have x2 : s2.mxcsr = s.mxcsr := o2.mxcsr + have z2 : s2.xmm .xmm13 = s.xmm .xmm13 := by rw [o2.xmm _ (by decide), xmm_setXmm]; rfl + have z2' : s2.xmm .xmm12 = s.xmm .xmm12 := by rw [o2.xmm _ (by decide), xmm_setXmm]; rfl + rw [xmm_setXmm, ifp rfl, xmm_setXmm, ifn (by decide), xmm_setXmm, ifn (by decide)] at h3 + generalize hV : csubV (montV (s.mem.readW (coeffAddr fP (4 * i)) 128) (s.xmm .xmm13) (s.xmm .xmm12)) = V at h3 + vrund [g2, m2, e2.1, e2.2, hdx, w0, x2, h3] + refine ⟨fun k hk => ?_, by rw [show (16 : BitVec 64) = BitVec.ofNat 64 (4 * 4) from rfl, coeffAddr_add, + Nat.mul_succ], (Frame.refl _ _).writeW (List.mem_singleton_self _) _ (pR_contains fP j0), ⟨?_, ?_⟩, ?_, ?_, + ⟨fun r hr => ?_, rfl, rfl⟩⟩ + · rw [coeffAt_write128 _ _ j0 _ (by omega)] + split + · rename_i h + rw [ifp (by omega), ← hV, dword_csubV _ (by omega), mulZ (hx _ (by omega)) (hz _ (by omega)) + (dword_montV ho (prod_lt hx hz) (by omega))] + dsimp only + rw [show 4 * i + (k - 4 * i) = k by omega, Fin.mul_comm] + · rename_i h + rw [hS k hk] + by_cases h' : k < 4 * i + · rw [ifp h', ifp (by omega)] + · rw [ifn h', ifn (by omega)] + · simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags]; exact c2.q + · simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags]; exact c2.qinv + · exact z2 + · exact z2' + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr.1, hr.2, ite_false] + +/-- Every coefficient times `8347681`. -/ +theorem vscale_ok {fP sP : Addr} (_hd : (pR sP).Disjoint (pR fP)) {G : Poly} (s : State) (hc : VConsts s) + (hdi : s.gpr .rdi = fP) (_hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP G) (_hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (_hw : pR sP ∈ s.wr) : + WP isa vscale s fun s' => PolyIs s'.mem fP (G.map (· * 8347681)) ∧ BInv fP s s' := by + refine WP.seq (WP.mono (Q := fun (w : State) => w.gpr .rdx = fP ∧ + w.xmm .xmm13 = shufDwords ((0 : BitVec 64) ++ BitVec.setWidth 64 16382#32) 0 ∧ + w.xmm .xmm12 = w.xmm .xmm13 ∧ VConsts w ∧ Keep [.rdx, .rax] s w ∧ w.mem = s.mem ∧ w.mxcsr = s.mxcsr) + (by + vrund [hdi, eval_movdqa] + refine ⟨⟨?_, ?_⟩, ⟨fun r hr => ?_, rfl, rfl⟩⟩ + · simp only [RegUpd.xmm_setReg, xmm_setXmm, reduceCtorEq, ite_false]; exact hc.q + · simp only [RegUpd.xmm_setReg, xmm_setXmm, reduceCtorEq, ite_false]; exact hc.qinv + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setXmm, hr.1, hr.2, ite_false]) fun w ⟨hdx, h13, h12, hcw, kw, mw, xw⟩ => ?_) + obtain ⟨hz, ho⟩ := scale_zlanes _ h13 + replace ho : ZOdd (w.xmm .xmm13) (w.xmm .xmm12) := by rw [h12]; exact ho + refine WP.mono (wp_rcxLoop (N := 64) (by decide) (by decide) + (fun i u => Scaled u.mem fP G i ∧ u.gpr .rdx = coeffAddr fP (4 * i) ∧ VConsts u ∧ + u.xmm .xmm13 = w.xmm .xmm13 ∧ u.xmm .xmm12 = w.xmm .xmm12 ∧ Keep [.rcx, .rdx] w u ∧ + Frame [pR fP] w.mem u.mem ∧ u.mxcsr = w.mxcsr) + (fun u o _ => ⟨fun k hk => by rw [o.mem, mw, ifn (by omega)]; exact polyIs_toNat hS (by rw [n_eq]; exact hk), + by rw [o.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + gonly_vconsts o hcw, by rw [o.xmm], by rw [o.xmm], o.keep.mono (by simp), by rw [o.mem]; exact Frame.refl _ _, + o.mxcsr⟩) + (fun i hi u ⟨hS', hdx', hc', hz', hzo', hk', hf', hx'⟩ => WP.mono (vscale_step hi hc' (by rw [hz']; exact hz) + (by rw [hz', hzo']; exact ho) hdx' hS' (by rw [hk'.2.2, kw.2.2]; exact hwf)) + fun u' ⟨hS'', hdx'', hf'', hc'', hz'', hzo'', hk'', hcx, hzf, hx''⟩ => + ⟨⟨hS'', hdx'', hc'', by rw [hz'', hz'], by rw [hzo'', hzo'], (hk'.trans hk'').mono (by simp), + hf'.trans hf'', by rw [hx'', hx']⟩, hcx, hzf⟩)) fun u ⟨hS', _, hc', _, _, hk', hf', hx'⟩ => ?_ + refine ⟨polyIs_of_toNat fun k hk => ?_, ⟨(kw.trans hk').mono (by simp), by rw [← mw]; exact hf', hc', + by rw [hx', xw]⟩⟩ + rw [n_eq] at hk + rw [hS' k hk, ifp (by omega), map_mul_get _ _ (by rw [n_eq]; exact hk)] theorem nttInv_correct (s : State) (hs : (inPlaceK nttInv).pre s) : ∃ t s', Exec isa Impl.MlDsa.X86_64.Arith.nttInv s t s' ∧ abiPreserved s s' ∧ (inPlaceK nttInv).post s s' := by - have hw : pR (s.gpr .rdi) ∈ s.wr := by rw [hs.2.1]; simp - have hz : pR (s.gpr .rsi) ∈ s.rd ++ s.wr := by rw [hs.1, hs.2.1]; simp - obtain ⟨t, s', he, ⟨hP, hf⟩, hk⟩ := WP.keep (c := Impl.MlDsa.X86_64.Arith.nttInv) - [.rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11] - (Q := fun s' => PolyIs s'.mem (s.gpr .rdi) (nttInv (polyAt s.mem (s.gpr .rdi))) ∧ - Frame [pR (s.gpr .rdi), pR (s.gpr .rsi)] s.mem s'.mem) - (WP.seq (WP.mono (Ntt.pro_ok hs negZetaTab (4 * 255) 255 (by decide)) fun s1 hI => - WP.seq (WP.mono (NttInv.lays_ok hw hz hs.2.2.1.symm nttInvLens _ 255 s1 (fun _ h => by - simp only [nttInvLens, nttLens, List.mem_cons, List.not_mem_nil, or_false] at h ⊢; omega) - NttInv.chain_inv hI) fun s2 ⟨k, hI2⟩ => - WP.mono (NttInv.scale_ok s2 hI2.rsi hI2.poly (by rw [hI2.wr]; exact hw)) fun s' ⟨hP, hf, _⟩ => - ⟨by rw [nttInv_eq_layers]; exact hP, hI2.frame.trans (hf.mono (by simp))⟩))) (by decide +kernel) - exact ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he (gprPreserved_of hk (by decide) hf - (by simpa using ⟨hs.2.2.2.1, hs.2.2.2.2.1⟩)), hP⟩ + have hd : (pR (s.gpr .rsi)).Disjoint (pR (s.gpr .rdi)) := hs.2.2.1.symm + refine mx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 => + WP.mono (nttBody_ok hs k1 f1 + (G := (nttInvLens.foldl nttInvLayer (polyAt s.mem (s.gpr .rdi))).map (· * 8347681)) + fun s2 hI hdi hsi hwf hw => ?_) fun s' ⟨hP, hf⟩ => ⟨by rw [nttInv_eq_layers]; exact hP, hf⟩ + simp only [nttInvLens, List.foldl_cons, List.foldl_nil] + refine LI.seq hdi hsi hwf hw hd (invLay1_ok hd) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay2_ok hd) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 4 63 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 8 31 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 16 15 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 32 7 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 64 3 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 128 1 (by decide) (by decide)) ?_ hI + exact fun _ hI => LI.last hdi hsi hwf hw hd (vscale_ok hd) hI theorem nttInv_ct : ConstantTime isa (inPlaceK nttInv).pre (inPlaceK nttInv).pub Impl.MlDsa.X86_64.Arith.nttInv := VG.Taint.constantTime (A := taint) (X86_64.Taint.ofRegs [.rdi, .rsi, .rsp]) inPlace_agree (by taint_decide) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttLoop.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttLoop.lean deleted file mode 100644 index eac150643..000000000 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttLoop.lean +++ /dev/null @@ -1,165 +0,0 @@ -import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttBfly -import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Table - -/-! -# ML-DSA on x86-64: the blocks and layers of `NTT` and `NTT⁻¹` - -Untrusted: everything here is checked by Lean. The loops of `nttBlk` and -`nttLay`, for any butterfly code that does what a butterfly `op` of the -specification does (`BflyOk`): a block runs `len` butterflies (`blockN`), -and a layer its `128 / len` blocks (`layerN`), with the zetas `Z (zi c)`, -whose values `tab` the table at `zP` holds. --/ - -namespace VG.Proof.MlDsa.X86_64.Arith - -open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith -open VG.Proof.MlDsa.Arith -open VG.Proof.MlKem.X86_64 (Keep WP.keep wp_countdown toNat_setWidth64) -open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs) - -/-- The table `tab` holds the values of the zetas `Z`. -/ -def TabOf (tab : Nat → Nat) (Z : Nat → Zq) : Prop := ∀ k < 256, tab k = (Z k).val - -/-! ## A block -/ - -section -variable {code : Nat → List Instr} {op : Poly → Nat → Nat → Zq → Poly} (hb : BflyOk code op) -include hb - -/-- The `len` butterflies of a block. -/ -theorem bflys_ok {fP : Addr} {len start : Nat} (hlen : 0 < len) (hs : start + 2 * len ≤ 256) (z : Zq) - (G : Poly) (s : State) (hsi : s.gpr .rsi = coeffAddr fP start) (h9 : s.gpr .r9 = BitVec.ofNat 64 z.val) - (hG : PolyIs s.mem fP G) (hw : pR fP ∈ s.wr) (hc : s.gpr .rcx = BitVec.ofNat 64 len) : - WP isa (.loop (.block (code len)) .ne) s fun s' => - PolyIs s'.mem fP (blockN op G len z start len) ∧ Frame [pR fP] s.mem s'.mem ∧ - s'.gpr .rsi = coeffAddr fP (start + len) ∧ Keep [.rax, .rdx, .rsi, .rcx, .r10, .r11] s s' := by - refine wp_countdown (cnt := .rcx) (N := len) (by omega) hlen (fun t s' => - PolyIs s'.mem fP (blockN op G len z start t) ∧ Frame [pR fP] s.mem s'.mem ∧ - s'.gpr .rsi = coeffAddr fP (start + t) ∧ Keep [.rax, .rdx, .rsi, .rcx, .r10, .r11] s s') - (fun t ht s' ⟨hP, hf, hs', hk⟩ _ => ?_) (fun _ h => h) ⟨hG, Frame.refl _ _, hsi, Keep.refl _ _⟩ hc - refine WP.mono (hb fP len (start + t) hlen (by omega) z _ s' hs' (by rw [hk.gpr (by decide), h9]) hP - (by rw [hk.2.2]; exact hw)) fun s'' ⟨⟨hP', hf', hsi', hcx, hz⟩, hk'⟩ => - ⟨⟨?_, hf.trans hf', by rw [hsi', hs', coeffAddr_succ, Nat.add_assoc], (hk.trans hk').mono (by decide)⟩, - hcx, hz⟩ - rw [blockN_succ]; exact hP' - -omit hb in -theorem blkPre_ok (len : Nat) (dz : BitVec 32) (s : State) (h : InRegions (s.rd ++ s.wr) (s.gpr .r8) 4) : - WP isa (.block [.mov32 .r9 (.mem (at_ .r8 0)), .alu .add .r8 (.imm dz), - .mov32 .rcx (.imm (BitVec.ofNat 32 len))]) s fun s' => - (s'.gpr .r9 = BitVec.setWidth 64 (s.mem.readW (s.gpr .r8) 32) ∧ s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ - s'.gpr .rcx = BitVec.setWidth 64 (BitVec.ofNat 32 len) ∧ s'.mem = s.mem) ∧ Keep [.r9, .r8, .rcx] s s' := by - refine WP.keep _ ?_ (by rfl) - xrund [h] - -omit hb in -theorem blkPost_ok (len : Nat) (hl : 4 * len < 2 ^ 31) (s : State) : - WP isa (.block [.alu .add .rsi (.imm (BitVec.ofNat 32 (4 * len))), .alu .sub .rdi (.imm 1)]) s fun s' => - (s'.gpr .rsi = s.gpr .rsi + BitVec.ofNat 64 (4 * len) ∧ s'.gpr .rdi = s.gpr .rdi - 1 ∧ - s'.zf = some (s.gpr .rdi - 1 == 0) ∧ s'.mem = s.mem) ∧ Keep [.rsi, .rdi] s s' := by - refine WP.keep _ ?_ (by rfl) - xrund [sx_ofNat hl] - -/-- A block, with the zeta `Z k` at `r8`. -/ -theorem blk_ok {tab : Nat → Nat} {Z : Nat → Zq} (hZ : TabOf tab Z) {fP zP : Addr} {len start k : Nat} - (hlen : 0 < len) (hl : len ≤ 128) (hs : start + 2 * len ≤ 256) - (hk : k < 256) (dz : BitVec 32) (G : Poly) (s : State) (hsi : s.gpr .rsi = coeffAddr fP start) - (h8 : s.gpr .r8 = coeffAddr zP k) (hG : PolyIs s.mem fP G) (hw : pR fP ∈ s.wr) - (hz : pR zP ∈ s.rd ++ s.wr) (ht : Tab tab s.mem zP 256) : - WP isa (nttBlk (code len) len dz) s fun s' => - (PolyIs s'.mem fP (blockN op G len (Z k) start len) ∧ Frame [pR fP] s.mem s'.mem ∧ - s'.gpr .rsi = coeffAddr fP (start + 2 * len) ∧ s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ - s'.gpr .rdi = s.gpr .rdi - 1 ∧ s'.zf = some (s.gpr .rdi - 1 == 0)) ∧ - Keep [.r9, .r8, .rcx, .rax, .rdx, .rsi, .rcx, .r10, .r11, .rsi, .rdi] s s' := by - refine WP.seq (WP.mono (blkPre_ok len dz s (by rw [h8]; exact ⟨_, hz, coeff_contains _ (show k < 256 by omega)⟩)) - fun s1 ⟨⟨h9, h8', hc, hm⟩, k1⟩ => ?_) - have hz9 : s1.gpr .r9 = BitVec.ofNat 64 (Z k).val := by - rw [h9, h8, ← coeffAt_eq, ht k hk, ← hZ k hk] - apply BitVec.eq_of_toNat_eq - rw [toNat_setWidth64, BitVec.toNat_ofNat, BitVec.toNat_ofNat] - have := val_lt (Z k) - rw [hZ k hk, Nat.mod_eq_of_lt (by omega), Nat.mod_eq_of_lt (by omega)] - refine WP.seq (WP.mono (bflys_ok hb (fP := fP) hlen hs (Z k) G s1 (by rw [k1.gpr (by decide), hsi]) hz9 - (by rw [hm]; exact hG) (by rw [k1.2.2]; exact hw) (by - rw [hc]; apply BitVec.eq_of_toNat_eq - rw [toNat_setWidth64, BitVec.toNat_ofNat, BitVec.toNat_ofNat]; omega)) fun s2 ⟨hP, hf, hsi2, k2⟩ => ?_) - refine WP.mono (blkPost_ok len (Nat.lt_of_le_of_lt (Nat.mul_le_mul_left 4 hl) (by decide)) s2) - fun s3 ⟨⟨hsi3, hdi, hz3, hm3⟩, k3⟩ => - ⟨⟨by rw [hm3]; exact hP, by rw [hm3, ← hm]; exact hf, ?_, ?_, ?_, ?_⟩, ((k1.trans k2).trans k3).mono (by decide)⟩ - · rw [hsi3, hsi2, coeffAddr_add, show start + len + len = start + 2 * len by omega] - · rw [k3.gpr (by decide), k2.gpr (by decide), h8'] - · rw [hdi, k2.gpr (by decide), k1.gpr (by decide)] - · rw [hz3, k2.gpr (by decide), k1.gpr (by decide)] - -/-! ## A layer -/ - -omit hb in -theorem layPre_ok (c : Nat) (hl : c < 2 ^ 31) (s : State) : - WP isa (.block [.mov32 .rdi (.imm (BitVec.ofNat 32 c))]) s fun s' => - (s'.gpr .rdi = BitVec.ofNat 64 c ∧ s'.mem = s.mem) ∧ Keep [.rdi] s s' := by - refine WP.keep _ ?_ (by rfl) - xrund - apply BitVec.eq_of_toNat_eq - rw [toNat_setWidth64, BitVec.toNat_ofNat, BitVec.toNat_ofNat]; omega - -omit hb in -theorem layPost_ok (s : State) : - WP isa (.block [.alu .sub .rsi (.imm 1024)]) s fun s' => - (s'.gpr .rsi = s.gpr .rsi - 1024 ∧ s'.mem = s.mem) ∧ Keep [.rsi] s s' := by - refine WP.keep _ ?_ (by rfl) - xrund [show BitVec.signExtend 64 (1024 : BitVec 32) = 1024 by decide] - -omit hb in -/-- The facts about the lengths of the layers. -/ -theorem lens_facts : ∀ len ∈ nttLens, 0 < len ∧ len ≤ 128 ∧ 2 * len * (128 / len) = 256 ∧ 0 < 128 / len := by - decide - -/-- A layer, from `rsi` = `f` and the zeta of its first block at `r8`. -/ -theorem lay_ok {tab : Nat → Nat} {Z : Nat → Zq} (hZ : TabOf tab Z) {fP zP : Addr} {len : Nat} - (hlen : len ∈ nttLens) (dz : BitVec 32) (zi : Nat → Nat) - (hzi : ∀ c < 128 / len, zi c < 256) - (hstep : ∀ c < 128 / len, coeffAddr zP (zi c) + BitVec.signExtend 64 dz = coeffAddr zP (zi (c + 1))) - (F : Poly) (s : State) (hsi : s.gpr .rsi = fP) (h8 : s.gpr .r8 = coeffAddr zP (zi 0)) - (hF : PolyIs s.mem fP F) (hw : pR fP ∈ s.wr) (hz : pR zP ∈ s.rd ++ s.wr) - (hd : (pR zP).Disjoint (pR fP)) (ht : Tab tab s.mem zP 256) : - WP isa (nttLay (code len) len dz) s fun s' => - (PolyIs s'.mem fP (layerN op F len (fun c => Z (zi c)) (128 / len)) ∧ Frame [pR fP] s.mem s'.mem ∧ - s'.gpr .rsi = fP ∧ s'.gpr .r8 = coeffAddr zP (zi (128 / len))) ∧ - Keep [.rdi, .r9, .r8, .rcx, .rax, .rdx, .rsi, .rcx, .r10, .r11, .rsi, .rdi, .rsi] s s' := by - obtain ⟨hl0, hl1, hl2, hl3⟩ := lens_facts len hlen - refine WP.seq (WP.mono (layPre_ok (128 / len) (by have := Nat.div_le_self 128 len; omega) s) - fun s1 ⟨⟨hdi, hm⟩, k1⟩ => ?_) - refine WP.seq (WP.mono (Q := fun (s' : State) => - PolyIs s'.mem fP (layerN op F len (fun c => Z (zi c)) (128 / len)) ∧ - Frame [pR fP] s.mem s'.mem ∧ s'.gpr .rsi = coeffAddr fP 256 ∧ s'.gpr .r8 = coeffAddr zP (zi (128 / len)) ∧ - Keep [.rdi, .r9, .r8, .rcx, .rax, .rdx, .rsi, .rcx, .r10, .r11, .rsi, .rdi] s s') ?_ - fun s2 ⟨hP, hf, hsi2, h82, k2⟩ => ?_) - · refine wp_countdown (cnt := .rdi) (N := 128 / len) (by have := Nat.div_le_self 128 len; omega) hl3 - (fun c (s' : State) => - PolyIs s'.mem fP (layerN op F len (fun c => Z (zi c)) c) ∧ Frame [pR fP] s.mem s'.mem ∧ - s'.gpr .rsi = coeffAddr fP (2 * len * c) ∧ s'.gpr .r8 = coeffAddr zP (zi c) ∧ - Keep [.rdi, .r9, .r8, .rcx, .rax, .rdx, .rsi, .rcx, .r10, .r11, .rsi, .rdi] s s') - (fun c hc s' ⟨hP, hf, hs', h8', hk⟩ _ => ?_) - (fun s' ⟨hP, hf, hs', h8', hk⟩ => ⟨hP, hf, by rw [hs', hl2], h8', hk⟩) - ⟨by rw [hm]; exact hF, by rw [hm]; exact Frame.refl _ _, by rw [k1.gpr (by decide), hsi]; simp, - by rw [k1.gpr (by decide), h8], k1.mono (by decide)⟩ hdi - have hcm : 2 * len * c + 2 * len ≤ 256 := by - have : 2 * len * (c + 1) ≤ 2 * len * (128 / len) := Nat.mul_le_mul_left _ (by omega) - rw [Nat.mul_succ] at this; omega - refine WP.mono (blk_ok hb hZ hl0 hl1 hcm (hzi c hc) dz _ s' hs' h8' hP (by rw [hk.2.2]; exact hw) - (by rw [hk.2.1, hk.2.2]; exact hz) (ht.frame hf (by simpa using hd) (by decide))) - fun s'' ⟨⟨hP', hf', hsi', h8'', hdi', hz'⟩, hk'⟩ => ⟨⟨by rw [layerN_succ]; exact hP', - hf.trans hf', ?_, ?_, (hk.trans hk').mono (by decide)⟩, hdi', hz'⟩ - · rw [hsi', Nat.mul_succ] - · rw [h8'', h8', hstep c hc] - · refine WP.mono (layPost_ok s2) fun s3 ⟨⟨hsi3, hm3⟩, k3⟩ => - ⟨⟨by rw [hm3]; exact hP, by rw [hm3]; exact hf, ?_, by rw [k3.gpr (by decide), h82]⟩, - (k2.trans k3).mono (by decide)⟩ - rw [hsi3, hsi2, coeffAddr] - show fP + BitVec.ofNat 64 1024 - 1024 = fP - exact BitVec.add_sub_cancel _ _ - -end - -end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Table.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Table.lean index 4501f89d9..fa3dc3c46 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Table.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Table.lean @@ -1,55 +1,22 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Basic -import VerifiedGarbage.Proof.Framework.Range /-! # ML-DSA on x86-64: tables of constants in the working space -Untrusted: everything here is checked by Lean. `storeTab t n` leaves the -`u32`s `t 0, …, t (n - 1)` at `r9` (`Tab`), and writes nothing else -(`storeTab_ok`). +Untrusted: everything here is checked by Lean. A table of `u32`s in the +working space (`Tab`), which writes elsewhere keep (`Tab.frame`). -/ namespace VG.Proof.MlDsa.X86_64.Arith open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith open VG.Proof.MlDsa.Arith -open VG.Proof.MlKem.X86_64 (Keep WP.keep ifp ifn) open VG.Spec.MlDsa (coeffAt) /-- The first `n` entries of the table `t` are the `u32`s at `p`. -/ def Tab (t : Nat → Nat) (m : Mem) (p : Addr) (n : Nat) : Prop := ∀ k < n, coeffAt m p k = BitVec.ofNat 32 (t k) -theorem tabStep_ok (t : Nat → Nat) (i : Nat) (s : State) - (hw : InRegions s.wr (s.gpr .r9 + BitVec.ofNat 64 (4 * i)) 4) : - WP isa (.block (tabStep t i)) s fun s' => - s'.mem = s.mem.writeW (s.gpr .r9 + BitVec.ofNat 64 (4 * i)) (BitVec.ofNat 32 (t i)) ∧ - Keep [.rax] s s' := by - refine WP.keep _ ?_ (by rfl) - unfold tabStep - xrund [hw] - -/-- The table, stored in the 1024 bytes at `r9`. -/ -theorem storeTab_ok (t : Nat → Nat) {n : Nat} (hn : n ≤ 256) (s : State) - (hw : pR (s.gpr .r9) ∈ s.wr) : - WP isa (.block (storeTab t n)) s fun s' => - Tab t s'.mem (s.gpr .r9) n ∧ Frame [pR (s.gpr .r9)] s.mem s'.mem ∧ Keep [.rax] s s' := by - refine WP.mono (wp_range_flatMap (M := isa) (fun k s' => Keep [.rax] s s' ∧ - Frame [pR (s.gpr .r9)] s.mem s'.mem ∧ Tab t s'.mem (s.gpr .r9) k) - (fun k s' hk ⟨hk', hf, ht⟩ => ?_) n (Nat.le_refl _) s - ⟨Keep.refl _ _, Frame.refl _ _, fun _ h => absurd h (Nat.not_lt_zero _)⟩) - fun s' ⟨hk, hf, ht⟩ => ⟨ht, hf, hk⟩ - have h9 : s'.gpr .r9 = s.gpr .r9 := hk'.gpr (by decide) - refine WP.mono (tabStep_ok t k s' (by - rw [hk'.2.2, h9]; exact ⟨_, hw, coeff_contains _ (show k < 256 by omega)⟩)) - fun s'' ⟨hm', hk''⟩ => ⟨(hk'.trans hk'').mono (by decide), ?_, fun j hj => ?_⟩ - · rw [hm', h9] - exact hf.writeW (List.mem_singleton_self _) _ (coeff_contains _ (show k < 256 by omega)) - · rw [hm', h9, ← coeffAddr, coeffAt_writeW _ _ (show j < 256 by omega) (show k < 256 by omega)] - by_cases e : k = j - · subst e; rw [ifp rfl] - · rw [ifn e]; exact ht j (by omega) - /-- Writes elsewhere keep the table. -/ theorem Tab.frame {t : Nat → Nat} {m m' : Mem} {p : Addr} {n : Nat} (h : Tab t m p n) {rs : List Region} (hf : Frame rs m m') (hd : ∀ r ∈ rs, (pR p).Disjoint r) (hn : n ≤ 256) : Tab t m' p n := diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VArith.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VArith.lean new file mode 100644 index 000000000..f8add0c32 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VArith.lean @@ -0,0 +1,285 @@ +import VerifiedGarbage.Proof.MlDsa.Arith.Mont +import VerifiedGarbage.Proof.Framework.X86_64.Avx +import VerifiedGarbage.Proof.MlKem.X86_64.VArith + +/-! +# ML-DSA on x86-64: arithmetic modulo `q` in doublewords + +Untrusted: everything here is checked by Lean. What `vmont`, `vcadd` and +`vcsub` (`Impl/MlDsa/X86_64/Arith/Vec.lean`) compute in each doubleword: + +* `montV d z zo`, the register `vmont` leaves: each doubleword is `mont` of + the product of those of `d` and `z` (`dword_montV`), if the even + doublewords of `zo` are the odd ones of `z` and each product is less than + `q · 2³²` (each quadword product `P` becomes `P + m · q`, which is + `mont P · 2³²`: `redc_toNat`); +* `caddL`, `csubL`: a doubleword plus `q` if it is negative, and less `q` + first, which `condSub` describes (`csubL_toNat`, `subD_toNat`, + `addD_toNat`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 +open VG.Proof.MlDsa.Arith +open VG.Spec.MlDsa (q) + +/-! ## Quadwords -/ + +theorem qword_app0 (a b : BitVec 64) : qword (a ++ b) 0 = b := by + apply BitVec.eq_of_getLsbD_eq; intro i hi + simp only [qword, BitVec.getLsbD_extractLsb', BitVec.getLsbD_append] + simp [hi] + +theorem qword_app1 (a b : BitVec 64) : qword (a ++ b) 1 = a := by + apply BitVec.eq_of_getLsbD_eq; intro i hi + simp only [qword, BitVec.getLsbD_extractLsb', BitVec.getLsbD_append] + simp [hi] + +theorem dword_lo (x : BitVec 128) (i : Nat) : dword x (2 * i) = (qword x i).extractLsb' 0 32 := by + apply BitVec.eq_of_getLsbD_eq; intro j hj + simp only [dword, qword, BitVec.getLsbD_extractLsb', hj, decide_true, Bool.true_and, + decide_eq_true (show 0 + j < 64 by omega)] + exact congrArg _ (by omega) + +theorem dword_hi (x : BitVec 128) (i : Nat) : dword x (2 * i + 1) = (qword x i).extractLsb' 32 32 := by + apply BitVec.eq_of_getLsbD_eq; intro j hj + simp only [dword, qword, BitVec.getLsbD_extractLsb', hj, decide_true, Bool.true_and, + decide_eq_true (show 32 + j < 64 by omega)] + exact congrArg _ (by omega) + +/-- The low doubleword of a quadword, zero-extended. -/ +def lo32 (x : BitVec 64) : BitVec 64 := (x.extractLsb' 0 32).setWidth 64 + +theorem lo32_toNat (x : BitVec 64) : (lo32 x).toNat = x.toNat % 2 ^ 32 := by + rw [lo32, BitVec.toNat_setWidth, BitVec.extractLsb'_toNat, Nat.shiftRight_zero, + Nat.mod_eq_of_lt (Nat.lt_of_lt_of_le (Nat.mod_lt _ (by decide)) (by decide))] + +theorem qword_paddq (x y : BitVec 128) {i : Nat} (hi : i < 2) : + qword (XBinOp.eval .paddq x y) i = qword x i + qword y i := by + rcases (by omega : i = 0 ∨ i = 1) with rfl | rfl + · simp only [XBinOp.eval, qword_app0] + · simp only [XBinOp.eval, qword_app1] + +theorem qword_pmuludq (x y : BitVec 128) {i : Nat} (hi : i < 2) : + qword (XBinOp.eval .pmuludq x y) i = lo32 (qword x i) * lo32 (qword y i) := by + rcases (by omega : i = 0 ∨ i = 1) with rfl | rfl + · simp only [XBinOp.eval, qword_app0, lo32, ← dword_lo] + · simp only [XBinOp.eval, qword_app1, lo32, ← dword_lo] + +theorem qword_psrlq32 (x : BitVec 128) {i : Nat} (hi : i < 2) : + qword (XShiftOp.eval .psrlq x 32) i = qword x i >>> 32 := by + rcases (by omega : i = 0 ∨ i = 1) with rfl | rfl + · simp only [XShiftOp.eval, show ¬ 63 < (32 : BitVec 8).toNat by decide, ite_false, qword_app0]; rfl + · simp only [XShiftOp.eval, show ¬ 63 < (32 : BitVec 8).toNat by decide, ite_false, qword_app1]; rfl + +theorem toNat_lo32_mul (x y : BitVec 64) : (lo32 x * lo32 y).toNat = x.toNat % 2 ^ 32 * (y.toNat % 2 ^ 32) := by + rw [BitVec.toNat_mul, lo32_toNat, lo32_toNat] + have h1 := Nat.mod_lt x.toNat (show 0 < 2 ^ 32 by decide) + have h2 := Nat.mod_lt y.toNat (show 0 < 2 ^ 32 by decide) + exact Nat.mod_eq_of_lt (Nat.lt_of_lt_of_le (Nat.mul_lt_mul'' h1 h2) (by decide)) + +/-! ## Montgomery reduction of a quadword -/ + +/-- `q` in each doubleword. -/ +def qV : BitVec 128 := 0x007FE001007FE001007FE001007FE001#128 + +/-- `-q⁻¹ mod 2³²` in each doubleword. -/ +def qinvV : BitVec 128 := 0xFC7FDFFFFC7FDFFFFC7FDFFFFC7FDFFF#128 + +theorem lo32_qword_qV {i : Nat} (hi : i < 2) : (lo32 (qword qV i)).toNat = q := by + rcases (by omega : i = 0 ∨ i = 1) with rfl | rfl <;> decide + +theorem lo32_qword_qinvV {i : Nat} (hi : i < 2) : (lo32 (qword qinvV i)).toNat = montQInv := by + rcases (by omega : i = 0 ∨ i = 1) with rfl | rfl <;> decide + +/-- `vredc` on a quadword `x`: `x + m · q`. -/ +def redc (x qi qq : BitVec 64) : BitVec 64 := x + lo32 (lo32 x * lo32 qi) * lo32 qq + +theorem redc_toNat {x qi qq : BitVec 64} (hqi : (lo32 qi).toNat = montQInv) (hqq : (lo32 qq).toNat = q) + (hx : x.toNat < q * 2 ^ 32) : (redc x qi qq).toNat = mont x.toNat * 2 ^ 32 := by + have hm : (lo32 (lo32 x * lo32 qi)).toNat = montM x.toNat := by + rw [lo32_toNat, BitVec.toNat_mul, lo32_toNat, hqi, montM, Nat.mod_mod_of_dvd _ (by decide)] + have hmq : (lo32 (lo32 x * lo32 qi) * lo32 qq).toNat = montM x.toNat * q := by + rw [BitVec.toNat_mul, hm, hqq] + exact Nat.mod_eq_of_lt (Nat.lt_of_lt_of_le (Nat.mul_lt_mul_of_pos_right (montM_lt _) (by decide)) + (by decide)) + rw [redc, BitVec.toNat_add, hmq, ← mont_mul] + have := mont_lt hx + rw [q_eq] at this + exact Nat.mod_eq_of_lt (by omega) + +theorem redc_hi {x qi qq : BitVec 64} (hqi : (lo32 qi).toNat = montQInv) (hqq : (lo32 qq).toNat = q) + (hx : x.toNat < q * 2 ^ 32) : ((redc x qi qq).extractLsb' 32 32).toNat = mont x.toNat := by + rw [BitVec.extractLsb'_toNat, redc_toNat hqi hqq hx, Nat.shiftRight_eq_div_pow, + Nat.mul_div_cancel _ (by decide)] + have := mont_lt hx + rw [q_eq] at this + exact Nat.mod_eq_of_lt (by omega) + +theorem redc_lo {x qi qq : BitVec 64} (hqi : (lo32 qi).toNat = montQInv) (hqq : (lo32 qq).toNat = q) + (hx : x.toNat < q * 2 ^ 32) : (redc x qi qq).extractLsb' 0 32 = 0 := by + apply BitVec.eq_of_toNat_eq + rw [BitVec.extractLsb'_toNat, redc_toNat hqi hqq hx, Nat.shiftRight_zero, Nat.mul_mod_left] + rfl + +theorem or_zero_toNat (x : BitVec 32) : (x ||| 0).toNat = x.toNat := by + simp + +theorem zero_or_toNat (x : BitVec 32) : ((0 : BitVec 32) ||| x).toNat = x.toNat := by + simp + +theorem lo_shr32 (x : BitVec 64) : ((x >>> 32).extractLsb' 0 32).toNat = (x.extractLsb' 32 32).toNat := by + rw [BitVec.extractLsb'_toNat, BitVec.extractLsb'_toNat, BitVec.toNat_ushiftRight, Nat.shiftRight_zero] + +theorem hi_shr32 (x : BitVec 64) : (x >>> 32).extractLsb' 32 32 = 0 := by + apply BitVec.eq_of_toNat_eq + rw [BitVec.extractLsb'_toNat, BitVec.toNat_ushiftRight, Nat.shiftRight_eq_div_pow, + Nat.shiftRight_eq_div_pow, Nat.div_div_eq_div_mul] + have := x.isLt + rw [Nat.div_eq_of_lt (by omega)]; rfl + +/-! ## `vmont` -/ + +/-- The register `vmont d z zo` leaves in `d`. -/ +def montV (d z zo : BitVec 128) : BitVec 128 := + let u := shufDwords d 0xF5 + let a := XBinOp.eval .pmuludq d z + let b := XBinOp.eval .pmuludq u zo + XBinOp.eval .por + (XShiftOp.eval .psrlq (XBinOp.eval .paddq a (XBinOp.eval .pmuludq (XBinOp.eval .pmuludq a qinvV) qV)) 32) + (XBinOp.eval .paddq b (XBinOp.eval .pmuludq (XBinOp.eval .pmuludq b qinvV) qV)) + +theorem qword_vredc (a : BitVec 128) {j : Nat} (hj : j < 2) : + qword (XBinOp.eval .paddq a (XBinOp.eval .pmuludq (XBinOp.eval .pmuludq a qinvV) qV)) j = + redc (qword a j) (qword qinvV j) (qword qV j) := by + rw [qword_paddq _ _ hj, qword_pmuludq _ _ hj, qword_pmuludq _ _ hj]; rfl + +theorem toNat_qword_pmuludq (x y : BitVec 128) {j : Nat} (hj : j < 2) : + (qword (XBinOp.eval .pmuludq x y) j).toNat = (dword x (2 * j)).toNat * (dword y (2 * j)).toNat := by + rw [qword_pmuludq _ _ hj, toNat_lo32_mul, dword_lo, dword_lo, BitVec.extractLsb'_toNat, + BitVec.extractLsb'_toNat, Nat.shiftRight_zero, Nat.shiftRight_zero] + +/-- Each doubleword of `montV d z zo` is `mont` of the product of those of +`d` and `z`. -/ +theorem dword_montV {d z zo : BitVec 128} (hzo : ∀ j < 2, dword zo (2 * j) = dword z (2 * j + 1)) + (hb : ∀ i < 4, (dword d i).toNat * (dword z i).toNat < q * 2 ^ 32) {i : Nat} (hi : i < 4) : + (dword (montV d z zo) i).toNat = mont ((dword d i).toNat * (dword z i).toNat) := by + have hqi : ∀ j < 2, (lo32 (qword qinvV j)).toNat = montQInv := fun j hj => lo32_qword_qinvV hj + have hqq : ∀ j < 2, (lo32 (qword qV j)).toNat = q := fun j hj => lo32_qword_qV hj + rw [montV, dword_por] + obtain ⟨j, hj, rfl | rfl⟩ : ∃ j < 2, i = 2 * j ∨ i = 2 * j + 1 := ⟨i / 2, by omega, by omega⟩ + all_goals + have hx : (qword (XBinOp.eval .pmuludq d z) j).toNat = (dword d (2 * j)).toNat * (dword z (2 * j)).toNat := + toNat_qword_pmuludq d z hj + have hy : (qword (XBinOp.eval .pmuludq (shufDwords d 0xF5) zo) j).toNat = + (dword d (2 * j + 1)).toNat * (dword z (2 * j + 1)).toNat := by + rw [toNat_qword_pmuludq _ _ hj, hzo j hj, dword_shufDwords _ _ (by omega)] + rcases (by omega : j = 0 ∨ j = 1) with rfl | rfl <;> rfl + · -- an even doubleword: the quotient of the even product, moved down + rw [dword_lo, dword_lo, qword_psrlq32 _ hj, qword_vredc _ hj, qword_vredc _ hj, + redc_lo (hqi j hj) (hqq j hj) (by rw [hy]; exact hb _ (by omega)), or_zero_toNat, lo_shr32, + redc_hi (hqi j hj) (hqq j hj) (by rw [hx]; exact hb _ (by omega)), hx] + · -- an odd doubleword: the quotient of the odd product, in place + rw [dword_hi, dword_hi, qword_psrlq32 _ hj, hi_shr32, zero_or_toNat, qword_vredc _ hj, + redc_hi (hqi j hj) (hqq j hj) (by rw [hy]; exact hb _ (by omega)), hy] + +/-! ## Conditional additions and subtractions of `q` -/ + +/-- `q` as a doubleword. -/ +def qB : BitVec 32 := 8380417#32 + +theorem dword_qV {i : Nat} (hi : i < 4) : dword qV i = qB := by + rcases cases4 hi with rfl | rfl | rfl | rfl <;> decide + +/-- `vcadd` on a doubleword: `d + q` if `d` is negative (as a signed doubleword). -/ +def caddL (d : BitVec 32) : BitVec 32 := d + (d.sshiftRight (min (31 : BitVec 8).toNat 32) &&& qB) + +/-- `vcsub` on a doubleword. -/ +def csubL (d : BitVec 32) : BitVec 32 := caddL (d - qB) + +theorem sshiftRight31 (d : BitVec 32) : + d.sshiftRight (min (31 : BitVec 8).toNat 32) = if d.toNat < 2 ^ 31 then 0 else -1 := by + rw [show min (31 : BitVec 8).toNat 32 = 31 from rfl] + apply BitVec.eq_of_toInt_eq + rw [BitVec.toInt_sshiftRight, MlKem.X86_64.W.toInt32] + have := d.isLt + split + · rw [show (0 : BitVec 32).toInt = 0 by decide, Int.shiftRight_eq_div_pow]; omega + · rw [show (-1 : BitVec 32).toInt = -1 by decide, Int.shiftRight_eq_div_pow]; omega + +theorem caddL_toNat (d : BitVec 32) : + (caddL d).toNat = if d.toNat < 2 ^ 31 then d.toNat else (d.toNat + q) % 2 ^ 32 := by + rw [caddL, sshiftRight31] + split + · rw [show (0 : BitVec 32) &&& qB = 0 by decide]; exact congrArg BitVec.toNat (BitVec.add_zero d) + · rw [show (-1 : BitVec 32) &&& qB = qB by decide, BitVec.toNat_add]; rfl + +/-- `vcsub` reduces a doubleword less than `2q`. -/ +theorem csubL_toNat {d : BitVec 32} (h : d.toNat < 2 * q) : (csubL d).toNat = condSub d.toNat := by + have e : (d - qB).toNat = (d.toNat + 2 ^ 32 - q) % 2 ^ 32 := by + rw [BitVec.toNat_sub]; rw [q_eq] at *; simp only [qB, BitVec.toNat_ofNat]; omega + rw [csubL, caddL_toNat, e, condSub]; rw [q_eq] at * + split <;> split <;> omega + +/-- The sum of two reduced doublewords, reduced by `vcsub`. -/ +theorem addD_toNat {a b : BitVec 32} (ha : a.toNat < q) (hb : b.toNat < q) : + (csubL (a + b)).toNat = condSub (a.toNat + b.toNat) := by + have e : (a + b).toNat = a.toNat + b.toNat := by + rw [BitVec.toNat_add]; rw [q_eq] at *; omega + rw [csubL_toNat (by rw [e]; omega), e] + +/-- The difference of two reduced doublewords, reduced by `vcadd`. -/ +theorem subD_toNat {a b : BitVec 32} (ha : a.toNat < q) (hb : b.toNat < q) : + (caddL (a - b)).toNat = condSub (a.toNat + q - b.toNat) := by + have e : (a - b).toNat = (a.toNat + 2 ^ 32 - b.toNat) % 2 ^ 32 := by + rw [BitVec.toNat_sub]; have := b.isLt; omega + rw [caddL_toNat, e, condSub]; rw [q_eq] at * + split <;> split <;> omega + +/-- `b - a + q` of two reduced doublewords, which `vibfly` multiplies by the zeta. -/ +theorem subq_toNat {a b : BitVec 32} (ha : a.toNat < q) (hb : b.toNat < q) : + (b - a + qB).toNat = b.toNat + q - a.toNat := by + rw [BitVec.toNat_add, BitVec.toNat_sub]; rw [q_eq] at *; simp only [qB, BitVec.toNat_ofNat]; omega + +/-! ## Butterflies, lane by lane -/ + +/-- A product by a zeta in Montgomery form, reduced: `ζ · y`. -/ +theorem mulZ {b z m : BitVec 32} {y ζ : Spec.MlDsa.Zq} (hb : b.toNat = y.val) (hz : z.toNat = ζ.val * 2 ^ 32 % q) + (hm : m.toNat = mont (b.toNat * z.toNat)) : (csubL m).toNat = (ζ * y).val := by + have hx : b.toNat * z.toNat < q * 2 ^ 32 := by + rw [hb, hz] + exact Nat.lt_of_lt_of_le (Nat.mul_lt_mul_of_lt_of_le (val_lt y) (Nat.le_of_lt (Nat.mod_lt _ (by decide))) + (by decide)) (by decide) + rw [csubL_toNat (by rw [hm]; exact mont_lt hx), hm, condSub_mont hx, hb, hz, mont_mulR, val_mul, + Nat.mul_comm] + +/-- The lanes of `vbfly`: `x + ζ · y` and `x - ζ · y`, from `t = ζ · y`. -/ +theorem bflyD {a t : BitVec 32} {x y ζ : Spec.MlDsa.Zq} (ha : a.toNat = x.val) (ht : t.toNat = (ζ * y).val) : + (csubL (a + t)).toNat = (x + ζ * y).val ∧ (caddL (a - t)).toNat = (x - ζ * y).val := by + have hx := val_lt x + have hzy := val_lt (ζ * y) + rw [addD_toNat (by rw [ha]; exact hx) (by rw [ht]; exact hzy), + subD_toNat (by rw [ha]; exact hx) (by rw [ht]; exact hzy), ha, ht, val_add, val_sub] + exact ⟨rfl, rfl⟩ + +/-- The lanes of `vibfly`: `x + y` and `ζ · (y - x)`. -/ +theorem ibflyD {a b z m : BitVec 32} {x y ζ : Spec.MlDsa.Zq} (ha : a.toNat = x.val) (hb : b.toNat = y.val) + (hz : z.toNat = ζ.val * 2 ^ 32 % q) (hm : m.toNat = mont ((b - a + qB).toNat * z.toNat)) : + (csubL (a + b)).toNat = (x + y).val ∧ (csubL m).toNat = (ζ * (y - x)).val := by + have hx := val_lt x + have hy := val_lt y + have e := subq_toNat (a := a) (b := b) (by rw [ha]; exact hx) (by rw [hb]; exact hy) + refine ⟨by rw [addD_toNat (by rw [ha]; exact hx) (by rw [hb]; exact hy), ha, hb, val_add], ?_⟩ + have hb' : (b - a + qB).toNat < q * 2 ^ 32 := by rw [e, ha, hb]; rw [q_eq] at *; omega + have hx' : (b - a + qB).toNat * z.toNat < q * 2 ^ 32 := by + rw [hz]; rw [e, ha, hb] at hb' ⊢ + have := Nat.mod_lt (ζ.val * 2 ^ 32) (show 0 < q by decide) + rw [q_eq] at * + exact Nat.lt_of_lt_of_le (Nat.mul_lt_mul_of_lt_of_le (show y.val + 8380417 - x.val < 2 * 8380417 by omega) + (Nat.le_of_lt this) (by decide)) (by decide) + rw [csubL_toNat (by rw [hm]; exact mont_lt hx'), hm, condSub_mont hx', hz, mont_mulR, e, ha, hb, + val_mul, val_sub', Nat.mul_mod_mod, Nat.mul_comm ζ.val, + Nat.add_sub_assoc (Nat.le_of_lt x.isLt) y.val] + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLanes.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLanes.lean new file mode 100644 index 000000000..a6e0c8ede --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLanes.lean @@ -0,0 +1,138 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VArith +import VerifiedGarbage.Proof.MlKem.X86_64.VLanes +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Ntt + +/-! +# ML-DSA on x86-64: coefficients in the doublewords of SSE registers + +Untrusted: everything here is checked by Lean. A register holds four +coefficients (`DLanes`), and the butterflies `vbfly` and `vibfly` compute +four butterflies of the specification at once (`vbfly_ok`, `vibfly_ok`), +from `q` and `-q⁻¹` in `xmm15` and `xmm14` (`VConsts`), with the zetas in +Montgomery form in `xmm13` (`ZLanes`) and its odd doublewords in the even +ones of `xmm12` (`ZOdd`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (XOnly XKeep xmm_setXmm mxcsr_setXmm ifp ifn) +open VG.Impl.MlKem.X86_64 (xb xmov) +open VG.Spec.MlDsa (q Zq) + +/-- The doublewords of `x` are the values of the coefficients `f 0, …, f 3`. -/ +def DLanes (x : BitVec 128) (f : Nat → Zq) : Prop := ∀ i < 4, (dword x i).toNat = (f i).val + +/-- The doublewords of `x` are the zetas `ζ i · 2³² mod q` (Montgomery form). -/ +def ZLanes (x : BitVec 128) (ζ : Nat → Zq) : Prop := ∀ i < 4, (dword x i).toNat = (ζ i).val * 2 ^ 32 % q + +/-- The even doublewords of `zo` are the odd ones of `z`. -/ +def ZOdd (z zo : BitVec 128) : Prop := ∀ j < 2, dword zo (2 * j) = dword z (2 * j + 1) + +/-- The constants of the vector code are in place. -/ +structure VConsts (s : State) : Prop where + q : s.xmm .xmm15 = qV + qinv : s.xmm .xmm14 = qinvV + +theorem VConsts.setXmm {s : State} (hc : VConsts s) {d : XReg} (h14 : XReg.xmm14 ≠ d) + (h15 : XReg.xmm15 ≠ d) (v : BitVec 128) : VConsts (s.setXmm d v) := + ⟨by rw [xmm_setXmm, ifn h15]; exact hc.q, by rw [xmm_setXmm, ifn h14]; exact hc.qinv⟩ + +theorem xonly_vconsts {rs : List XReg} {s s' : State} (h : XOnly rs s s') (hc : VConsts s) + (h14 : XReg.xmm14 ∉ rs) (h15 : XReg.xmm15 ∉ rs) : VConsts s' := + ⟨by rw [h.xmm _ h15, hc.q], by rw [h.xmm _ h14, hc.qinv]⟩ + +/-! ## Registers -/ + +/-- `vcadd` on a register. -/ +def caddV (d : BitVec 128) : BitVec 128 := + XBinOp.eval .paddd d (XBinOp.eval .pand (XShiftOp.eval .psrad d 31) qV) + +/-- `vcsub` on a register. -/ +def csubV (d : BitVec 128) : BitVec 128 := caddV (XBinOp.eval .psubd d qV) + +theorem dword_psubd (a b : BitVec 128) {i : Nat} (hi : i < 4) : + dword (XBinOp.eval .psubd a b) i = dword a i - dword b i := by + rcases cases4 hi with rfl | rfl | rfl | rfl <;> simp [XBinOp.eval] + +theorem dword_pand (a b : BitVec 128) (i : Nat) : + dword (XBinOp.eval .pand a b) i = dword a i &&& dword b i := by + apply BitVec.eq_of_getLsbD_eq; intro j hj + simp [XBinOp.eval, dword, hj] + +theorem dword_psrad (a : BitVec 128) (n : BitVec 8) {i : Nat} (hi : i < 4) : + dword (XShiftOp.eval .psrad a n) i = (dword a i).sshiftRight (min n.toNat 32) := by + rcases cases4 hi with rfl | rfl | rfl | rfl <;> simp [XShiftOp.eval] + +theorem dword_caddV (d : BitVec 128) {i : Nat} (hi : i < 4) : dword (caddV d) i = caddL (dword d i) := by + rw [caddV, dword_paddd _ _ hi, dword_pand, dword_psrad _ _ hi, dword_qV hi]; rfl + +theorem dword_csubV (d : BitVec 128) {i : Nat} (hi : i < 4) : dword (csubV d) i = csubL (dword d i) := by + rw [csubV, dword_caddV _ hi, dword_psubd _ _ hi, dword_qV hi]; rfl + +/-- A reduced coefficient times a zeta in Montgomery form is less than `q · 2³²`. -/ +theorem prod_lt {y z : BitVec 128} {f ζ : Nat → Zq} (hy : DLanes y f) (hz : ZLanes z ζ) : + ∀ i < 4, (dword y i).toNat * (dword z i).toNat < q * 2 ^ 32 := fun i hi => by + rw [hy i hi, hz i hi] + exact Nat.lt_of_lt_of_le (Nat.mul_lt_mul_of_lt_of_le (val_lt (f i)) (Nat.le_of_lt (Nat.mod_lt _ (by decide))) + (by decide)) (by decide) + +theorem subq_mul_lt {a b c : Nat} (ha : a < 8380417) (hb : b < 8380417) (hc : c < q) : + (b + q - a) * c < q * 2 ^ 32 := by + rw [q_eq] at * + exact Nat.lt_of_lt_of_le (Nat.mul_lt_mul_of_lt_of_le (show b + 8380417 - a < 2 * 8380417 by omega) + (Nat.le_of_lt hc) (by decide)) (by decide) + +theorem eval_movdqa (a b : BitVec 128) : XBinOp.eval .movdqa a b = b := rfl + +/-! ## Butterflies -/ + +theorem vbfly_ok {s : State} (hc : VConsts s) {x y ζ : Nat → Zq} (hx : DLanes (s.xmm .xmm0) x) + (hy : DLanes (s.xmm .xmm1) y) (hz : ZLanes (s.xmm .xmm13) ζ) (ho : ZOdd (s.xmm .xmm13) (s.xmm .xmm12)) : + WP isa (.block vbfly) s fun s' => DLanes (s'.xmm .xmm0) (fun i => x i + ζ i * y i) ∧ + DLanes (s'.xmm .xmm3) (fun i => x i - ζ i * y i) ∧ + XOnly [.xmm1, .xmm2, .xmm4, .xmm0, .xmm3] s s' := by + simp only [vbfly, vmont, vredc, vcsub, vcadd, xmov, xb, List.cons_append, List.nil_append] + vrun [eval_movdqa] + rw [hc.q, hc.qinv] + refine ⟨?_, ?_, by xonly⟩ + · change DLanes (csubV (XBinOp.eval .paddd (s.xmm .xmm0) + (csubV (montV (s.xmm .xmm1) (s.xmm .xmm13) (s.xmm .xmm12))))) _ + intro i hi + rw [dword_csubV _ hi, dword_paddd _ _ hi, dword_csubV _ hi] + exact (bflyD (hx i hi) (mulZ (hy i hi) (hz i hi) (dword_montV ho (prod_lt hy hz) hi))).1 + · change DLanes (caddV (XBinOp.eval .psubd (s.xmm .xmm0) + (csubV (montV (s.xmm .xmm1) (s.xmm .xmm13) (s.xmm .xmm12))))) _ + intro i hi + rw [dword_caddV _ hi, dword_psubd _ _ hi, dword_csubV _ hi] + exact (bflyD (hx i hi) (mulZ (hy i hi) (hz i hi) (dword_montV ho (prod_lt hy hz) hi))).2 + +theorem vibfly_ok {s : State} (hc : VConsts s) {x y ζ : Nat → Zq} (hx : DLanes (s.xmm .xmm0) x) + (hy : DLanes (s.xmm .xmm1) y) (hz : ZLanes (s.xmm .xmm13) ζ) (ho : ZOdd (s.xmm .xmm13) (s.xmm .xmm12)) : + WP isa (.block vibfly) s fun s' => DLanes (s'.xmm .xmm0) (fun i => x i + y i) ∧ + DLanes (s'.xmm .xmm3) (fun i => ζ i * (y i - x i)) ∧ + XOnly [.xmm1, .xmm2, .xmm4, .xmm0, .xmm3] s s' := by + simp only [vibfly, vmont, vredc, vcsub, vcadd, xmov, xb, List.cons_append, List.nil_append] + vrun [eval_movdqa] + rw [hc.q, hc.qinv] + have e : ∀ i < 4, dword (XBinOp.eval .paddd (XBinOp.eval .psubd (s.xmm .xmm1) (s.xmm .xmm0)) qV) i = + dword (s.xmm .xmm1) i - dword (s.xmm .xmm0) i + qB := fun i hi => by + rw [dword_paddd _ _ hi, dword_psubd _ _ hi, dword_qV hi] + have hb : ∀ i < 4, (dword (XBinOp.eval .paddd (XBinOp.eval .psubd (s.xmm .xmm1) (s.xmm .xmm0)) qV) i).toNat * + (dword (s.xmm .xmm13) i).toNat < q * 2 ^ 32 := fun i hi => by + rw [e i hi, subq_toNat (by rw [hx i hi]; exact val_lt _) (by rw [hy i hi]; exact val_lt _), hx i hi, hy i hi, + hz i hi] + exact subq_mul_lt (val_lt (x i)) (val_lt (y i)) (Nat.mod_lt _ (by decide)) + refine ⟨?_, ?_, by xonly⟩ + · change DLanes (csubV (XBinOp.eval .paddd (s.xmm .xmm0) (s.xmm .xmm1))) _ + intro i hi + rw [dword_csubV _ hi, dword_paddd _ _ hi, addD_toNat (by rw [hx i hi]; exact val_lt _) + (by rw [hy i hi]; exact val_lt _), hx i hi, hy i hi, val_add] + · change DLanes (csubV (montV (XBinOp.eval .paddd (XBinOp.eval .psubd (s.xmm .xmm1) (s.xmm .xmm0)) qV) + (s.xmm .xmm13) (s.xmm .xmm12))) _ + intro i hi + rw [dword_csubV _ hi] + exact (ibflyD (hx i hi) (hy i hi) (hz i hi) (by rw [dword_montV ho hb hi, e i hi])).2 + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLay.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLay.lean new file mode 100644 index 000000000..698814fa2 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLay.lean @@ -0,0 +1,322 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VMem +import VerifiedGarbage.Proof.MlKem.X86_64.VLay + +/-! +# ML-DSA on x86-64: the layers of the NTT and its inverse with `len ≥ 4` + +Untrusted: everything here is checked by Lean. For any butterfly code `bf` +that does what `op` does to the doublewords of two registers (`VBflyOk`), +and any block of the specification whose butterflies do `op` (`BlkOk`): +four butterflies of a block (`vstep`), the `len / 4` of them of a block +(`vblock_ok`), and the `128 / len` blocks of a layer (`vlay_ok`), on the +polynomial at `fP`, with the zetas from the table at `sP`. +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (XOnly xmm_setXmm ifp ifn sel sel_lt sel_zero add_ofNat_zero Keep wp_countdown + GOnly wp_rcxLoop sx1 sx16) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs zetas) + +/-- Runs a block of general-purpose and SSE instructions. -/ +syntax "vrund" (" [" Lean.Parser.Tactic.simpLemma,* "]")? : tactic +macro_rules + | `(tactic| vrund) => `(tactic| vrund []) + | `(tactic| vrund [$ls,*]) => `(tactic| vrunm [ea_atD, $ls,*]) + +/-- `GOnly` of a chain of `setReg` and `setFlags`. -/ +macro "gonlyd" : tactic => `(tactic| exact ⟨⟨fun r hr => by + simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false], rfl, rfl⟩, rfl, rfl, rfl⟩) + +theorem gonly_vconsts {rs : List Reg} {s s' : State} (h : GOnly rs s s') (hc : VConsts s) : VConsts s' := + ⟨by rw [h.xmm]; exact hc.q, by rw [h.xmm]; exact hc.qinv⟩ + +/-- The code `bf` of four butterflies does what `op` does to each pair of +doublewords of `xmm0` and `xmm1`, with the zetas in `xmm13` (and `xmm12`), +leaving the results in `xmm0` and `xmm3`. -/ +def VBflyOk (bf : List Instr) (op : Zq → Zq → Zq → Zq × Zq) : Prop := + ∀ s : State, VConsts s → ∀ x y ζ : Nat → Zq, DLanes (s.xmm .xmm0) x → DLanes (s.xmm .xmm1) y → + ZLanes (s.xmm .xmm13) ζ → ZOdd (s.xmm .xmm13) (s.xmm .xmm12) → + WP isa (.block bf) s fun s' => DLanes (s'.xmm .xmm0) (fun i => (op (x i) (y i) (ζ i)).1) ∧ + DLanes (s'.xmm .xmm3) (fun i => (op (x i) (y i) (ζ i)).2) ∧ + XOnly [.xmm1, .xmm2, .xmm4, .xmm0, .xmm3] s s' + +theorem vbfly_spec : VBflyOk vbfly (fun x y z => (x + z * y, x - z * y)) := + fun _ hc _ _ _ hx hy hz ho => vbfly_ok hc hx hy hz ho + +theorem vibfly_spec : VBflyOk vibfly (fun x y z => (x + y, z * (y - x))) := + fun _ hc _ _ _ hx hy hz ho => vibfly_ok hc hx hy hz ho + +/-- The first `t` butterflies of a block of the specification, with the +zeta of index `k` of the table, do `op` to `(j, j + len)`. -/ +structure BlkOk (blk : Poly → Nat → Nat → Nat → Nat → Poly) (op : Zq → Zq → Zq → Zq × Zq) : Prop where + zero : ∀ f len k st, blk f len k st 0 = f + add : ∀ f len k st t t', blk f len k st (t + t') = blk (blk f len k st t) len k (st + t) t' + get : ∀ f len k st t, 0 < len → t ≤ len → st + len + t ≤ n → ∀ i < n, + (blk f len k st t)[i]! = if st ≤ i ∧ i < st + t then (op f[i]! f[i + len]! (zetas k)).1 + else if st + len ≤ i ∧ i < st + len + t then (op f[i - len]! f[i]! (zetas k)).2 else f[i]! + +theorem blockN_add (op : Poly → Nat → Nat → Zq → Poly) (f : Poly) (len : Nat) (z : Zq) (st t t' : Nat) : + blockN op f len z st (t + t') = blockN op (blockN op f len z st t) len z (st + t) t' := by + simp only [blockN]; rw [← List.foldl_append, List.range'_append_1] + +/-- `blockN_bfly_get`, for the butterflies of the block up to `start + t` +only. -/ +theorem blockN_bfly_get' (w : Poly) {len : Nat} {z : Zq} {start t : Nat} (hlen : 0 < len) (ht : t ≤ len) + (hs : start + len + t ≤ n) {i : Nat} (hi : i < n) : + (blockN bfly w len z start t)[i]! = + if start ≤ i ∧ i < start + t then w[i]! + z * w[i + len]! + else if start + len ≤ i ∧ i < start + len + t then w[i - len]! - z * w[i]! + else w[i]! := by + induction t generalizing i with + | zero => + rw [blockN_zero, ite_eq_right (by omega), ite_eq_right (by omega)] + | succ t ih => + rw [blockN_succ, bfly_get _ hlen (by omega) _ hi, ih (i := start + t) (by omega) (by omega) (by omega), + ih (i := start + t + len) (by omega) (by omega) (by omega), ih (by omega) (by omega) hi] + rcases (by omega : i < start ∨ (start ≤ i ∧ i < start + t) ∨ i = start + t ∨ + (start + t < i ∧ i < start + len) ∨ (start + len ≤ i ∧ i < start + t + len) ∨ + i = start + t + len ∨ start + t + len < i) with h | h | rfl | h | h | rfl | h <;> + simp (disch := omega) only [ite_eq_left, ite_eq_right, Nat.add_sub_cancel, ↓reduceIte] + +/-- `blockN_bflyInv_get`, for the butterflies of the block up to +`start + t` only. -/ +theorem blockN_bflyInv_get' (w : Poly) {len : Nat} {z : Zq} {start t : Nat} (hlen : 0 < len) (ht : t ≤ len) + (hs : start + len + t ≤ n) {i : Nat} (hi : i < n) : + (blockN bflyInv w len z start t)[i]! = + if start ≤ i ∧ i < start + t then w[i]! + w[i + len]! + else if start + len ≤ i ∧ i < start + len + t then z * (w[i - len]! - w[i]!) + else w[i]! := by + induction t generalizing i with + | zero => + rw [blockN_zero, ite_eq_right (by omega), ite_eq_right (by omega)] + | succ t ih => + rw [blockN_succ, bflyInv_get _ hlen (by omega) _ hi, ih (i := start + t) (by omega) (by omega) (by omega), + ih (i := start + t + len) (by omega) (by omega) (by omega), ih (by omega) (by omega) hi] + rcases (by omega : i < start ∨ (start ≤ i ∧ i < start + t) ∨ i = start + t ∨ + (start + t < i ∧ i < start + len) ∨ (start + len ≤ i ∧ i < start + t + len) ∨ + i = start + t + len ∨ start + t + len < i) with h | h | rfl | h | h | rfl | h <;> + simp (disch := omega) only [ite_eq_left, ite_eq_right, Nat.add_sub_cancel, ↓reduceIte] + +theorem nttBlk_ok : BlkOk (fun f len k st t => blockN bfly f len (zetas k) st t) + (fun x y z => (x + z * y, x - z * y)) := + ⟨fun _ _ _ _ => rfl, fun _ _ _ _ _ _ => blockN_add _ _ _ _ _ _ _, + fun f _ _ _ _ hl ht hs _ hi => blockN_bfly_get' f hl ht hs hi⟩ + +/-- Algorithm 42 multiplies by `-ζ`; `vibfly` by `ζ`, the other way round. -/ +theorem neg_mul_sub (z x y : Zq) : -z * (x - y) = z * (y - x) := by + grind + +theorem nttInvBlk_ok : BlkOk (fun f len k st t => blockN bflyInv f len (-zetas k) st t) + (fun x y z => (x + y, z * (y - x))) := + ⟨fun _ _ _ _ => rfl, fun _ _ _ _ _ _ => blockN_add _ _ _ _ _ _ _, + fun f _ _ _ _ hl ht hs _ hi => by rw [blockN_bflyInv_get' f hl ht hs hi, neg_mul_sub]⟩ + +/-! ## A block -/ + +theorem f_in {rs : List Region} {fP : Addr} (hw : pR fP ∈ rs) {j : Nat} (hj : j + 4 ≤ 256) : + InRegions rs (coeffAddr fP j) 16 := + ⟨_, hw, pR_contains fP hj⟩ + +theorem tab_in {rs : List Region} {sP : Addr} (hw : pR sP ∈ rs) {k : Nat} (hk : k + 4 ≤ 256) : + InRegions rs (coeffAddr sP k) 16 := + ⟨_, hw, pR_contains sP hk⟩ + +/-- The facts a block keeps. -/ +structure BInv (fP : Addr) (s₀ s : State) : Prop where + keep : Keep [.r8, .rcx, .rdx, .rax] s₀ s + frame : Frame [pR fP] s₀.mem s.mem + consts : VConsts s + mxcsr : s.mxcsr = s₀.mxcsr + +theorem BInv.trans {fP : Addr} {s₁ s₂ s₃ : State} (h₁ : BInv fP s₁ s₂) (h₂ : BInv fP s₂ s₃) : BInv fP s₁ s₃ := + ⟨(h₁.keep.trans h₂.keep).mono (by simp), h₁.frame.trans h₂.frame, h₂.consts, h₂.mxcsr.trans h₁.mxcsr⟩ + +/-! ## Four butterflies -/ + +section +variable {bf : List Instr} {op : Zq → Zq → Zq → Zq × Zq} (hbf : VBflyOk bf op) + {blk : Poly → Nat → Nat → Nat → Nat → Poly} (hblk : BlkOk blk op) +include hbf hblk + +/-- The body of the loop over the vectors of a block. -/ +abbrev vbody (bf : List Instr) (len : Nat) : List Instr := + [.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm1 (at_ .rdx (4 * len))] ++ bf ++ + [.movdquStore (at_ .rdx 0) .xmm0, .movdquStore (at_ .rdx (4 * len)) .xmm3, .alu .add .rdx (.imm 16)] ++ + [.alu .sub .rcx (.imm 1)] + +theorem vstep {fP : Addr} {len st u k : Nat} (hl : 0 < len) (hs : st + 2 * len ≤ 256) (hu : 4 * u + 4 ≤ len) + {G : Poly} {s : State} (hc : VConsts s) (hz : ZLanes (s.xmm .xmm13) (fun _ => zetas k)) + (ho : ZOdd (s.xmm .xmm13) (s.xmm .xmm12)) + (hdx : s.gpr .rdx = coeffAddr fP (st + 4 * u)) (hS : PolyIs s.mem fP (blk G len k st (4 * u))) + (hw : pR fP ∈ s.wr) : + WP isa (.block (vbody bf len)) s fun s' => + PolyIs s'.mem fP (blk G len k st (4 * (u + 1))) ∧ s'.gpr .rdx = coeffAddr fP (st + 4 * (u + 1)) ∧ + Frame [pR fP] s.mem s'.mem ∧ VConsts s' ∧ s'.xmm .xmm13 = s.xmm .xmm13 ∧ + s'.xmm .xmm12 = s.xmm .xmm12 ∧ Keep [.rdx, .rcx] s s' ∧ + s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) ∧ s'.mxcsr = s.mxcsr := by + have j0 : st + 4 * u + 4 ≤ 256 := by omega + have j1 : st + 4 * u + len + 4 ≤ 256 := by omega + have a1 : coeffAddr fP (st + 4 * u) + BitVec.ofNat 64 (4 * len) = coeffAddr fP (st + 4 * u + len) := + coeffAddr_add _ _ _ + have r0 : InRegions (s.rd ++ s.wr) (coeffAddr fP (st + 4 * u)) 16 := f_in (List.mem_append_right _ hw) j0 + have r1 : InRegions (s.rd ++ s.wr) (coeffAddr fP (st + 4 * u + len)) 16 := f_in (List.mem_append_right _ hw) j1 + have w0 := f_in hw j0 + have w1 := f_in hw j1 + rw [vbody, List.append_assoc, List.append_assoc, WP.block_append_iff] + vrund [hdx, a1, r0, r1] + rw [WP.block_append_iff] + have hx := dlanes_load hS j0 + have hy := dlanes_load hS j1 + refine WP.mono (hbf _ ((hc.setXmm (by decide) (by decide) _).setXmm (by decide) (by decide) _) + (fun e => (blk G len k st (4 * u))[st + 4 * u + e]!) (fun e => (blk G len k st (4 * u))[st + 4 * u + len + e]!) + (fun _ => zetas k) (by rw [xmm_setXmm, xmm_setXmm]; exact hx) (by rw [xmm_setXmm]; exact hy) + (by rw [xmm_setXmm, xmm_setXmm]; exact hz) (by rw [xmm_setXmm, xmm_setXmm, xmm_setXmm, xmm_setXmm]; exact ho)) + fun s2 ⟨l0, l3, o2⟩ => ?_ + have c2 := xonly_vconsts o2 ((hc.setXmm (by decide) (by decide) _).setXmm (by decide) (by decide) _) (by decide) + (by decide) + have g2 : s2.gpr = s.gpr := o2.gpr + have m2 : s2.mem = s.mem := o2.mem + have e2 : s2.rd = s.rd ∧ s2.wr = s.wr := ⟨o2.rd, o2.wr⟩ + have x2 : s2.mxcsr = s.mxcsr := o2.mxcsr + have z2 : s2.xmm .xmm13 = s.xmm .xmm13 := by rw [o2.xmm _ (by decide), xmm_setXmm, xmm_setXmm]; rfl + have z2' : s2.xmm .xmm12 = s.xmm .xmm12 := by rw [o2.xmm _ (by decide), xmm_setXmm, xmm_setXmm]; rfl + vrund [g2, m2, e2.1, e2.2, hdx, a1, w0, w1, x2] + refine ⟨?_, ?_, ?_, ⟨c2.q, c2.qinv⟩, z2, z2', ⟨fun r hr => ?_, rfl, rfl⟩⟩ + · refine polyIs_write2 hS j0 j1 (by omega) l0 l3 fun i hi => ?_ + rw [show 4 * (u + 1) = 4 * u + 4 by omega, hblk.add, hblk.get _ _ _ _ _ hl (by omega) + (by rw [n_eq]; omega) _ (by rw [n_eq]; exact hi)] + by_cases c1 : st + 4 * u ≤ i ∧ i < st + 4 * u + 4 + · rw [ite_eq_left_of_eq_true _ _ (eq_true c1), ite_eq_left_of_eq_true _ _ (eq_true c1), + show st + 4 * u + (i - (st + 4 * u)) = i by omega, + show st + 4 * u + len + (i - (st + 4 * u)) = i + len by omega] + · rw [ite_eq_right_of_eq_false _ _ (eq_false c1), ite_eq_right_of_eq_false _ _ (eq_false c1)] + by_cases c2 : st + 4 * u + len ≤ i ∧ i < st + 4 * u + len + 4 + · rw [ite_eq_left_of_eq_true _ _ (eq_true c2), ite_eq_left_of_eq_true _ _ (eq_true (by omega)), + show st + 4 * u + (i - (st + 4 * u + len)) = i - len by omega, + show st + 4 * u + len + (i - (st + 4 * u + len)) = i by omega] + · rw [ite_eq_right_of_eq_false _ _ (eq_false c2), ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + · rw [show (16 : BitVec 64) = BitVec.ofNat 64 (4 * 4) from rfl, coeffAddr_add, + show st + 4 * u + 4 = st + 4 * (u + 1) by omega] + · exact frame_write2 (Frame.refl _ _) j0 j1 _ _ + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr.1, hr.2, ite_false] + +/-- The code of a block of a layer with `len ≥ 4`. -/ +abbrev vblk (bf : List Instr) (len : Nat) (dz : BitVec 32) : Prog isa := + .seq (.block (vzeta 0 ++ [.alu .add .r8 (.imm dz)])) + (.seq (VG.Impl.MlKem.X86_64.rcxLoop (len / 4) ([.movdquLoad .xmm0 (at_ .rdx 0), + .movdquLoad .xmm1 (at_ .rdx (4 * len))] ++ + bf ++ [.movdquStore (at_ .rdx 0) .xmm0, .movdquStore (at_ .rdx (4 * len)) .xmm3, + .alu .add .rdx (.imm 16)])) + (.block [.alu .add .rdx (.imm (BitVec.ofNat 32 (4 * len))), .alu .sub .rax (.imm 1)])) + +theorem vblock_ok {fP sP : Addr} {len st kz : Nat} (h4 : 4 ≤ len) (hl4 : len % 4 = 0) (hl : len ≤ 128) + (hs : st + 2 * len ≤ 256) (hkz : kz + 4 ≤ 256) (dz : BitVec 32) {G : Poly} {s : State} (hc : VConsts s) + (hdx : s.gpr .rdx = coeffAddr fP st) (h8r : s.gpr .r8 = coeffAddr sP kz) (hS : PolyIs s.mem fP G) + (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (vblk bf len dz) s fun s' => PolyIs s'.mem fP (blk G len kz st len) ∧ + s'.gpr .rdx = coeffAddr fP (st + 2 * len) ∧ s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ + s'.gpr .rax = s.gpr .rax - 1 ∧ s'.zf = some (s.gpr .rax - 1 == 0) ∧ BInv fP s s' := by + -- the zeta + refine WP.seq ?_ + rw [WP.block_append_iff] + refine WP.mono (vzeta_ok 0 (k := kz) (fun j _ => by rw [sel_zero]; omega) h8r + (tab_in (List.mem_append_right _ hw) hkz) hT) fun s1 ⟨z1, zo1, o1⟩ => ?_ + have g1 : s1.gpr = s.gpr := o1.gpr + refine WP.mono (Q := fun (s2 : State) => s2.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ + GOnly [.r8] s1 s2) + (by vrund [g1]; gonlyd) + fun s2 ⟨h82, o2⟩ => ?_ + have c2 := gonly_vconsts o2 (xonly_vconsts o1 hc (by decide) (by decide)) + have z2 : ZLanes (s2.xmm .xmm13) (fun _ => zetas kz) := by + rw [o2.xmm]; intro i hi; rw [z1 i hi]; dsimp only; rw [sel_zero, Nat.add_zero] + have zo2 : ZOdd (s2.xmm .xmm13) (s2.xmm .xmm12) := by rw [o2.xmm]; exact zo1 + have dx2 : s2.gpr .rdx = coeffAddr fP st := by rw [o2.keep.gpr (by decide), g1, hdx] + have hw2 : pR fP ∈ s2.wr := by rw [o2.keep.2.2, o1.wr]; exact hwf + have m2 : s2.mem = s.mem := by rw [o2.mem, o1.mem] + refine WP.seq (WP.mono (wp_rcxLoop (N := len / 4) (by omega) (by omega) + (fun u w => PolyIs w.mem fP (blk G len kz st (4 * u)) ∧ w.gpr .rdx = coeffAddr fP (st + 4 * u) ∧ + VConsts w ∧ w.xmm .xmm13 = s2.xmm .xmm13 ∧ w.xmm .xmm12 = s2.xmm .xmm12 ∧ Keep [.rcx, .rdx] s2 w ∧ + Frame [pR fP] s2.mem w.mem ∧ w.mxcsr = s2.mxcsr) + (fun w o hc => ⟨by rw [hblk.zero, o.mem, m2]; exact hS, by rw [o.keep.gpr (by decide), dx2]; rfl, + gonly_vconsts o c2, by rw [o.xmm], by rw [o.xmm], o.keep.mono (by simp), by rw [o.mem]; exact Frame.refl _ _, + o.mxcsr⟩) + (fun u hu w ⟨hS', hdx', hc', hz', hzo', hk', hf', hx'⟩ => WP.mono (vstep hbf hblk (by omega) hs (by + have := Nat.div_mul_cancel (Nat.dvd_of_mod_eq_zero hl4); omega) hc' (by rw [hz']; exact z2) + (by rw [hz', hzo']; exact zo2) hdx' hS' (by rw [hk'.2.2]; exact hw2)) + fun w' ⟨hS'', hdx'', hf'', hc'', hz'', hzo'', hk'', hcx, hzf, hx''⟩ => + ⟨⟨hS'', hdx'', hc'', by rw [hz'', hz'], by rw [hzo'', hzo'], (hk'.trans hk'').mono (by simp), + hf'.trans hf'', by rw [hx'', hx']⟩, hcx, hzf⟩)) fun w ⟨hS3, hdx3, hc3, _, _, hk3, hf3, hx3⟩ => ?_) + rw [show 4 * (len / 4) = len from Nat.mul_div_cancel' (Nat.dvd_of_mod_eq_zero hl4)] at hS3 hdx3 + have hax : w.gpr .rax = s.gpr .rax := by rw [hk3.gpr (by decide), o2.keep.gpr (by decide), g1] + have h8w : w.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz := by rw [hk3.gpr (by decide), h82] + vrund [hdx3, sx_ofNat (show 4 * len < 2 ^ 31 by omega), hax, h8w] + refine ⟨hS3, by rw [coeffAddr_add, show st + len + len = st + 2 * len by omega], ?_⟩ + have k1 : Keep [.r8, .rcx, .rdx, .rax] s w := + (Keep.trans (⟨fun r _ => by rw [g1], o1.rd, o1.wr⟩ : Keep [] s s1) (o2.keep.trans hk3)).mono (by simp) + refine ⟨⟨fun r hr => ?_, k1.2.1, k1.2.2⟩, by rw [← m2]; exact hf3, + ⟨by simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags]; exact hc3.q, + by simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags]; exact hc3.qinv⟩, + by simp only [RegUpd.mxcsr_setReg, RegUpd.mxcsr_setFlags]; rw [hx3, o2.mxcsr, o1.mxcsr]⟩ + simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false] + exact k1.gpr (by simp [hr]) + +/-! ## A layer -/ + +/-- The first `b` blocks of the layer with `len`, block `c` with the zeta of +index `zi c`. -/ +def layF (blk : Poly → Nat → Nat → Nat → Nat → Poly) (F : Poly) (len : Nat) (zi : Nat → Nat) (b : Nat) : + Poly := + (List.range b).foldl (fun f c => blk f len (zi c) (2 * len * c) len) F + +theorem vlay_ok {fP sP : Addr} {len k : Nat} (hlen : len ∈ [4, 8, 16, 32, 64, 128]) (dz : BitVec 32) + (zi : Nat → Nat) (hz0 : zi 0 = k) (hzi : ∀ c < 128 / len, zi c + 4 ≤ 256) + (hstep : ∀ c < 128 / len, coeffAddr sP (zi c) + BitVec.signExtend 64 dz = coeffAddr sP (zi (c + 1))) + {F : Poly} {s : State} (hc : VConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) + (hd : (pR sP).Disjoint (pR fP)) : + WP isa (vlay bf len k dz) s fun s' => PolyIs s'.mem fP (layF blk F len zi (128 / len)) ∧ + BInv fP s s' := by + have hl : 4 ≤ len ∧ len % 4 = 0 ∧ len ≤ 128 ∧ 2 * len * (128 / len) = 256 ∧ 0 < 128 / len ∧ + 128 / len ≤ 32 := by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hlen + rcases hlen with rfl | rfl | rfl | rfl | rfl | rfl <;> decide + obtain ⟨h4, hl4, hl128, hcov, hpos, h32⟩ := hl + have hk : k + 4 ≤ 256 := hz0 ▸ hzi 0 hpos + refine WP.seq (WP.mono (Q := fun (w : State) => w.gpr .rdx = fP ∧ w.gpr .r8 = coeffAddr sP k ∧ + w.gpr .rax = BitVec.ofNat 64 (128 / len) ∧ GOnly [.rdx, .r8, .rax] s w) + (by + simp only [leaR] + vrund [sx_ofNat (show 4 * k < 2 ^ 31 by omega), hsi, hdi] + refine ⟨?_, by gonlyd⟩ + apply BitVec.eq_of_toNat_eq + rw [BitVec.toNat_setWidth, BitVec.toNat_ofNat, BitVec.toNat_ofNat] + omega) fun w ⟨hdx, h8r, hax, o⟩ => ?_) + have hwf' : pR fP ∈ w.wr := by rw [o.keep.2.2]; exact hwf + have hw' : pR sP ∈ w.wr := by rw [o.keep.2.2]; exact hw + refine WP.mono (wp_countdown (cnt := .rax) (N := 128 / len) (by omega) hpos + (fun c u => PolyIs u.mem fP (layF blk F len zi c) ∧ u.gpr .rdx = coeffAddr fP (2 * len * c) ∧ + u.gpr .r8 = coeffAddr sP (zi c) ∧ BInv fP w u ∧ Tab zmTab u.mem sP 256) + (fun c hc u ⟨hS', hdx', h8', hb', hT'⟩ _ => ?_) (fun u h => h) + ⟨by rw [o.mem]; exact hS, by rw [hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [h8r, hz0], ⟨Keep.refl _ _, Frame.refl _ _, gonly_vconsts o hc, rfl⟩, by rw [o.mem]; exact hT⟩ hax) + fun u ⟨hS', _, _, hb', _⟩ => ⟨hS', ⟨(o.keep.trans hb'.keep).mono (by simp), + by rw [← o.mem]; exact hb'.frame, hb'.consts, by rw [hb'.mxcsr, o.mxcsr]⟩⟩ + have hs : 2 * len * c + 2 * len ≤ 256 := by + have : 2 * len * (c + 1) ≤ 2 * len * (128 / len) := Nat.mul_le_mul_left _ (by omega) + rw [Nat.mul_succ] at this; omega + refine WP.mono (vblock_ok hbf hblk h4 hl4 hl128 hs (hzi c hc) dz hb'.consts hdx' h8' hS' hT' + (by rw [hb'.keep.2.2]; exact hwf') (by rw [hb'.keep.2.2]; exact hw')) + fun u' ⟨hS'', hdx'', h8'', hax'', hzf'', hb''⟩ => + ⟨⟨by rw [layF, foldl_range_succ]; exact hS'', by rw [hdx'', Nat.mul_succ], + by rw [h8'', h8', hstep c hc], hb'.trans hb'', + hT'.frame hb''.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide)⟩, hax'', hzf''⟩ + +end + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLay21.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLay21.lean new file mode 100644 index 000000000..288058286 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VLay21.lean @@ -0,0 +1,458 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay + +/-! +# ML-DSA on x86-64: the layers of the NTT and its inverse with `len` = 2 and 1 + +Untrusted: everything here is checked by Lean. The layer with `len = 2` runs +two blocks at a time (`vstep2`): the lower halves of their coefficients +gathered into `xmm0` and the upper ones into `xmm1` by `punpcklqdq` and +`punpckhqdq`, and back. The layer with `len = 1` runs four blocks at a time +(`vstep1`): their coefficients gathered by `pshufd` and `punpck{l,h}qdq`, +and interleaved back by `punpck{l,h}dq`. +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (XOnly xmm_setXmm mxcsr_setXmm ifp ifn sel sel_lt add_ofNat_zero Keep GOnly + wp_rcxLoop sx32) +open VG.Impl.MlKem.X86_64 (xb xmov) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs zetas) + +theorem dword_punpcklqdq (a b : BitVec 128) {j : Nat} (hj : j < 4) : + dword (XBinOp.eval .punpcklqdq a b) j = if j < 2 then dword a j else dword b (j - 2) := by + rw [punpcklqdq_eq] + rcases cases4 hj with rfl | rfl | rfl | rfl <;> + simp (disch := decide) only [Nat.reduceLT, Nat.reduceSub, ite_true, ite_false, dword_ofDwords_0, + dword_ofDwords_1, dword_ofDwords_2, dword_ofDwords_3] + +theorem dword_punpckhqdq (a b : BitVec 128) {j : Nat} (hj : j < 4) : + dword (XBinOp.eval .punpckhqdq a b) j = if j < 2 then dword a (2 + j) else dword b j := by + rw [punpckhqdq_eq] + rcases cases4 hj with rfl | rfl | rfl | rfl <;> + simp (disch := decide) only [Nat.reduceLT, Nat.reduceAdd, ite_true, ite_false, dword_ofDwords_0, + dword_ofDwords_1, dword_ofDwords_2, dword_ofDwords_3] + +theorem dword_punpckldq' (a b : BitVec 128) {j : Nat} (hj : j < 4) : + dword (XBinOp.eval .punpckldq a b) j = if j % 2 = 0 then dword a (j / 2) else dword b (j / 2) := by + rw [dword_punpckldq] + rcases cases4 hj with rfl | rfl | rfl | rfl <;> simp + +theorem dword_punpckhdq' (a b : BitVec 128) {j : Nat} (hj : j < 4) : + dword (XBinOp.eval .punpckhdq a b) j = if j % 2 = 0 then dword a (2 + j / 2) else dword b (2 + j / 2) := by + rw [dword_punpckhdq] + rcases cases4 hj with rfl | rfl | rfl | rfl <;> simp + +/-- The doublewords of `x` that `pshufd` with `0xD8` puts in place `e`: the +even ones in the lower half, the odd ones in the upper half. -/ +theorem dword_d8 (x : BitVec 128) {e : Nat} (he : e < 4) : + dword (shufDwords x 0xD8) e = dword x (if e < 2 then 2 * e else 2 * (e - 2) + 1) := by + rw [dword_shufDwords _ _ he] + rcases cases4 he with rfl | rfl | rfl | rfl <;> rfl + +/-- The general-purpose registers but `rs`, memory, the permissions and +MXCSR are as they were. -/ +structure GKeep (rs : List Reg) (s s' : State) : Prop where + keep : Keep rs s s' + mem : s'.mem = s.mem + mxcsr : s'.mxcsr = s.mxcsr + +/-! ## The layer with `len = 2` -/ + +section +variable {bf : List Instr} {op : Zq → Zq → Zq → Zq × Zq} (hbf : VBflyOk bf op) + {blk : Poly → Nat → Nat → Nat → Nat → Poly} (hblk : BlkOk blk op) +include hbf hblk + +/-- The loads, the zetas and the gathering of the lower and upper halves. -/ +abbrev pre2 (o : BitVec 8) (dz : BitVec 32) : List Instr := + [.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm1 (at_ .rdx 16)] ++ vzeta o ++ + [.alu .add .r8 (.imm dz), xmov .xmm2 .xmm0, xb .punpcklqdq .xmm0 .xmm1, xb .punpckhqdq .xmm2 .xmm1, + xmov .xmm1 .xmm2] + +/-- The interleaving back, the stores and the counts. -/ +abbrev post2 : List Instr := + [xmov .xmm1 .xmm0, xb .punpcklqdq .xmm0 .xmm3, xb .punpckhqdq .xmm1 .xmm3, + .movdquStore (at_ .rdx 0) .xmm0, .movdquStore (at_ .rdx 16) .xmm1, .alu .add .rdx (.imm 32), + .alu .sub .rcx (.imm 1)] + +theorem vstep2 {fP sP : Addr} {i kz : Nat} (hi : i < 32) (o : BitVec 8) (dz : BitVec 32) (zi : Nat → Nat) + (hk : kz + 4 ≤ 256) (hsel : ∀ e < 4, kz + sel o e = zi (2 * i + e / 2)) + {F : Poly} {s : State} (hc : VConsts s) (hdx : s.gpr .rdx = coeffAddr fP (8 * i)) + (h8 : s.gpr .r8 = coeffAddr sP kz) (hS : PolyIs s.mem fP (layF blk F 2 zi (2 * i))) + (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (.block (pre2 o dz ++ (bf ++ post2))) s fun s' => + PolyIs s'.mem fP (layF blk F 2 zi (2 * (i + 1))) ∧ s'.gpr .rdx = coeffAddr fP (8 * (i + 1)) ∧ + s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ + s'.zf = some (s.gpr .rcx - 1 == 0) ∧ BInv fP s s' := by + have j0 : 8 * i + 4 ≤ 256 := by omega + have j1 : 8 * i + 4 + 4 ≤ 256 := by omega + have a1 : coeffAddr fP (8 * i) + BitVec.ofNat 64 16 = coeffAddr fP (8 * i + 4) := coeffAddr_add _ _ 4 + have r0 := f_in (List.mem_append_right s.rd hwf) j0 + have r1 := f_in (List.mem_append_right s.rd hwf) j1 + have hk' : ∀ j < 4, kz + sel o j < 256 := fun j _ => by have := sel_lt o j; omega + generalize hG : layF blk F 2 zi (2 * i) = G at hS + have lx := dlanes_load hS j0 + have ly := dlanes_load hS j1 + rw [WP.block_append_iff, show pre2 o dz = [.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm1 (at_ .rdx 16)] ++ + (vzeta o ++ [.alu .add .r8 (.imm dz), xmov .xmm2 .xmm0, xb .punpcklqdq .xmm0 .xmm1, + xb .punpckhqdq .xmm2 .xmm1, xmov .xmm1 .xmm2]) by simp, WP.block_append_iff] + vrund [hdx, a1, r0, r1] + rw [WP.block_append_iff] + refine WP.mono (vzeta_ok o hk' (by simp only [RegUpd.gpr_setXmm]; exact h8) + (by simp only [RegUpd.rd_setXmm, RegUpd.wr_setXmm]; exact tab_in (List.mem_append_right _ hw) hk) + (by simp only [RegUpd.mem_setXmm]; exact hT)) fun s1 ⟨z1, zo1, o1⟩ => ?_ + refine WP.mono (Q := fun (s1' : State) => DLanes (s1'.xmm .xmm0) (fun e => G[8 * i + e + 2 * (e / 2)]!) ∧ + DLanes (s1'.xmm .xmm1) (fun e => G[8 * i + 2 + e + 2 * (e / 2)]!) ∧ + ZLanes (s1'.xmm .xmm13) (fun e => zetas (zi (2 * i + e / 2))) ∧ ZOdd (s1'.xmm .xmm13) (s1'.xmm .xmm12) ∧ + VConsts s1' ∧ s1'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ GKeep [.r8] s s1') ?_ + fun s1' ⟨l0, l1, l13, l12, c1, h81, o1'⟩ => ?_ + · have g1 : s1.gpr = s.gpr := by rw [o1.gpr]; rfl + have m1 : s1.mem = s.mem := by rw [o1.mem]; rfl + have e1 : s1.rd = s.rd ∧ s1.wr = s.wr := ⟨by rw [o1.rd]; rfl, by rw [o1.wr]; rfl⟩ + have x1 : s1.mxcsr = s.mxcsr := by rw [o1.mxcsr]; rfl + have x0 : s1.xmm .xmm0 = s.mem.readW (coeffAddr fP (8 * i)) 128 := by + rw [o1.xmm _ (by decide)]; simp only [xmm_setXmm]; rfl + have x1' : s1.xmm .xmm1 = s.mem.readW (coeffAddr fP (8 * i + 4)) 128 := by + rw [o1.xmm _ (by decide)]; simp only [xmm_setXmm]; rfl + have c1 : VConsts s1 := xonly_vconsts o1 ((hc.setXmm (by decide) (by decide) _).setXmm (by decide) + (by decide) _) (by decide) (by decide) + vrund [g1, m1, e1.1, e1.2, x1, eval_movdqa] + refine ⟨?_, ?_, ?_, ?_, ⟨?_, ?_⟩, ⟨⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_⟩⟩ + · intro e he + rw [dword_punpcklqdq _ _ he, x0, x1'] + split + · rw [lx e he]; dsimp only; rw [show 8 * i + e + 2 * (e / 2) = 8 * i + e by omega] + · rw [ly (e - 2) (by omega)]; dsimp only + rw [show 8 * i + e + 2 * (e / 2) = 8 * i + 4 + (e - 2) by omega] + · intro e he + rw [dword_punpckhqdq _ _ he, x0, x1'] + split + · rw [lx (2 + e) (by omega)]; dsimp only + rw [show 8 * i + 2 + e + 2 * (e / 2) = 8 * i + (2 + e) by omega] + · rw [ly e he]; dsimp only; rw [show 8 * i + 2 + e + 2 * (e / 2) = 8 * i + 4 + e by omega] + · intro e he + rw [z1 e he]; dsimp only; rw [hsel e he] + · exact zo1 + · exact c1.q + · exact c1.qinv + · simp only [RegUpd.gpr_setXmm, RegUpd.gpr_setReg, RegUpd.gpr_setFlags, g1] + rw [ifn (by simpa using hr)] + all_goals simp only [RegUpd.rd_setXmm, RegUpd.wr_setXmm, RegUpd.mem_setXmm, mxcsr_setXmm, + RegUpd.rd_setReg, RegUpd.wr_setReg, RegUpd.mem_setReg, RegUpd.mxcsr_setReg, RegUpd.rd_setFlags, + RegUpd.wr_setFlags, RegUpd.mem_setFlags, RegUpd.mxcsr_setFlags, e1.1, e1.2, m1, x1] + rw [WP.block_append_iff] + refine WP.mono (hbf _ c1 _ _ _ l0 l1 l13 l12) fun s2 ⟨a0, a3, o2⟩ => ?_ + have g2 : s2.gpr .rdx = s.gpr .rdx := by rw [o2.gpr, o1'.keep.gpr (by decide)] + have e2 : s2.rd = s.rd ∧ s2.wr = s.wr := ⟨by rw [o2.rd, o1'.keep.2.1], by rw [o2.wr, o1'.keep.2.2]⟩ + have m2 : s2.mem = s.mem := by rw [o2.mem, o1'.mem] + have w0 := f_in hwf j0 + have w1 := f_in hwf j1 + have c2 := xonly_vconsts o2 c1 (by decide) (by decide) + simp only [post2, xmov, xb] + vrund [g2, e2.1, e2.2, m2, hdx, a1, w0, w1, sx32] + have r81 : s2.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz := by rw [o2.gpr, h81] + have rc1 : s2.gpr .rcx = s.gpr .rcx := by rw [o2.gpr, o1'.keep.gpr (by decide)] + refine ⟨?_, by rw [show (32 : BitVec 64) = BitVec.ofNat 64 (4 * 8) from rfl, coeffAddr_add, Nat.mul_succ], r81, + by rw [rc1], by rw [rc1], ?_⟩ + · -- the coefficients stored + refine polyIs_write2 hS j0 j1 (by omega) + (a := fun e => if e < 2 then (op G[8 * i + e]! G[8 * i + 2 + e]! (zetas (zi (2 * i)))).1 + else (op G[8 * i + (e - 2)]! G[8 * i + 2 + (e - 2)]! (zetas (zi (2 * i)))).2) + (b := fun e => if e < 2 then (op G[8 * i + 4 + e]! G[8 * i + 6 + e]! (zetas (zi (2 * i + 1)))).1 + else (op G[8 * i + 4 + (e - 2)]! G[8 * i + 6 + (e - 2)]! (zetas (zi (2 * i + 1)))).2) + (fun e he => ?_) (fun e he => ?_) (fun j hj => ?_) + · rw [dword_punpcklqdq _ _ he] + split + · rw [a0 e he]; dsimp only + rw [ite_eq_left (by omega), show 8 * i + e + 2 * (e / 2) = 8 * i + e by omega, + show 8 * i + 2 + e + 2 * (e / 2) = 8 * i + 2 + e by omega, show 2 * i + e / 2 = 2 * i by omega] + · rw [a3 (e - 2) (by omega)]; dsimp only + rw [ite_eq_right (by omega), show 8 * i + (e - 2) + 2 * ((e - 2) / 2) = 8 * i + (e - 2) by omega, + show 8 * i + 2 + (e - 2) + 2 * ((e - 2) / 2) = 8 * i + 2 + (e - 2) by omega, + show 2 * i + (e - 2) / 2 = 2 * i by omega] + · rw [dword_punpckhqdq _ _ he, eval_movdqa] + split + · rw [a0 (2 + e) (by omega)]; dsimp only + rw [ite_eq_left (by omega), show 8 * i + (2 + e) + 2 * ((2 + e) / 2) = 8 * i + 4 + e by omega, + show 8 * i + 2 + (2 + e) + 2 * ((2 + e) / 2) = 8 * i + 6 + e by omega, + show 2 * i + (2 + e) / 2 = 2 * i + 1 by omega] + · rw [a3 e he]; dsimp only + rw [ite_eq_right (by omega), show 8 * i + e + 2 * (e / 2) = 8 * i + 4 + (e - 2) by omega, + show 8 * i + 2 + e + 2 * (e / 2) = 8 * i + 6 + (e - 2) by omega, + show 2 * i + e / 2 = 2 * i + 1 by omega] + · -- the specification: two blocks + rw [← hG, show 2 * (i + 1) = 2 * i + 1 + 1 by omega, layF, foldl_range_succ, foldl_range_succ, ← layF, + hG, show 2 * 2 * (2 * i) = 8 * i by omega, show 2 * 2 * (2 * i + 1) = 8 * i + 4 by omega] + have hn : ∀ j, j < 256 → j < n := fun j h => by rw [n_eq]; exact h + rw [hblk.get _ _ _ _ _ (by decide) (by decide) (by rw [n_eq]; omega) _ (hn j hj)] + have p2 := fun j (h : j < 256) => hblk.get G 2 (zi (2 * i)) (8 * i) 2 (by decide) (by decide) + (by rw [n_eq]; omega) j (hn j h) + rcases (by omega : j < 8 * i ∨ (8 * i ≤ j ∧ j < 8 * i + 2) ∨ (8 * i + 2 ≤ j ∧ j < 8 * i + 4) ∨ + (8 * i + 4 ≤ j ∧ j < 8 * i + 6) ∨ (8 * i + 6 ≤ j ∧ j < 8 * i + 8) ∨ 8 * i + 8 ≤ j) with + h | h | h | h | h | h + · simp (disch := omega) only [ite_eq_left, ite_eq_right, p2 j hj] + · simp (disch := omega) only [ite_eq_left, ite_eq_right, p2 j hj] + rw [show 8 * i + (j - 8 * i) = j by omega, show 8 * i + 2 + (j - 8 * i) = j + 2 by omega] + · simp (disch := omega) only [ite_eq_left, ite_eq_right, p2 j hj] + rw [show 8 * i + (j - 8 * i - 2) = j - 2 by omega, show 8 * i + 2 + (j - 8 * i - 2) = j by omega] + · simp (disch := omega) only [ite_eq_left, ite_eq_right, p2 j hj, p2 (j + 2) (by omega)] + rw [show 8 * i + 4 + (j - (8 * i + 4)) = j by omega, + show 8 * i + 6 + (j - (8 * i + 4)) = j + 2 by omega] + · simp (disch := omega) only [ite_eq_left, ite_eq_right, p2 j hj, p2 (j - 2) (by omega)] + rw [show 8 * i + 4 + (j - (8 * i + 4) - 2) = j - 2 by omega, + show 8 * i + 6 + (j - (8 * i + 4) - 2) = j by omega] + · simp (disch := omega) only [ite_eq_left, ite_eq_right, p2 j hj] + · -- what the step keeps + refine ⟨⟨fun r hr => ?_, by simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags], + by simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags]⟩, frame_write2 (Frame.refl _ _) j0 j1 _ _, ⟨?_, ?_⟩, + by simp only [RegUpd.mxcsr_setReg, RegUpd.mxcsr_setFlags]; rw [o2.mxcsr, o1'.mxcsr]⟩ + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false] + rw [o2.gpr, o1'.keep.gpr (by simp [hr])] + · simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags, xmm_setXmm, reduceCtorEq, ite_false] + exact c2.q + · simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags, xmm_setXmm, reduceCtorEq, ite_false] + exact c2.qinv + +omit hbf hblk in +/-- The prologue of the layers with `len` = 2 and 1. -/ +theorem vpre21 {fP sP : Addr} (k : Nat) (hk : k + 4 ≤ 256) {s : State} (hdi : s.gpr .rdi = fP) + (hsi : s.gpr .rsi = sP) : + WP isa (.block (([.mov .rdx (.reg .rdi)] : List Instr) ++ leaR .r8 .rsi (4 * k))) s fun w => + w.gpr .rdx = fP ∧ w.gpr .r8 = coeffAddr sP k ∧ GOnly [.rdx, .r8] s w := by + simp only [leaR] + vrund [sx_ofNat (show 4 * k < 2 ^ 31 by omega), hsi, hdi] + gonlyd + +theorem vlay2_ok {fP sP : Addr} (k : Nat) (o : BitVec 8) (dz : BitVec 32) (zi kz : Nat → Nat) (hkz0 : kz 0 = k) + (hk : ∀ i < 32, kz i + 4 ≤ 256) (hsel : ∀ i < 32, ∀ e < 4, kz i + sel o e = zi (2 * i + e / 2)) + (hstep : ∀ i < 32, coeffAddr sP (kz i) + BitVec.signExtend 64 dz = coeffAddr sP (kz (i + 1))) + {F : Poly} {s : State} (hc : VConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) + (hd : (pR sP).Disjoint (pR fP)) : + WP isa (vlay2 bf k o dz) s fun s' => PolyIs s'.mem fP (layF blk F 2 zi 64) ∧ BInv fP s s' := by + have hk0 : k + 4 ≤ 256 := by have := hk 0 (by decide); omega + refine WP.seq (WP.mono (vpre21 k hk0 hdi hsi) fun w ⟨hdx, h8, og⟩ => ?_) + refine WP.mono (wp_rcxLoop (N := 32) (by decide) (by decide) + (fun i u => PolyIs u.mem fP (layF blk F 2 zi (2 * i)) ∧ u.gpr .rdx = coeffAddr fP (8 * i) ∧ + u.gpr .r8 = coeffAddr sP (kz i) ∧ BInv fP w u) + (fun u ou _ => ⟨by rw [ou.mem, og.mem]; exact hS, + by rw [ou.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [ou.keep.gpr (by decide), h8, hkz0], ⟨ou.keep.mono (by simp), by rw [ou.mem]; exact Frame.refl _ _, + gonly_vconsts ou (gonly_vconsts og hc), ou.mxcsr⟩⟩) + (fun i hi u ⟨hS', hdx', h8', hb'⟩ => ?_)) fun u ⟨hS', _, _, hb'⟩ => + ⟨hS', ⟨(og.keep.trans hb'.keep).mono (by simp), by rw [← og.mem]; exact hb'.frame, hb'.consts, + by rw [hb'.mxcsr, og.mxcsr]⟩⟩ + have hT' : Tab zmTab u.mem sP 256 := (by rw [og.mem]; exact hT : Tab zmTab w.mem sP 256).frame hb'.frame + (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide) + have hwf' : pR fP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hwf + have hw' : pR sP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hw + rw [show [Instr.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm1 (at_ .rdx 16)] ++ vzeta o ++ + [.alu .add .r8 (.imm dz), xmov .xmm2 .xmm0, xb .punpcklqdq .xmm0 .xmm1, xb .punpckhqdq .xmm2 .xmm1, + xmov .xmm1 .xmm2] ++ bf ++ [xmov .xmm1 .xmm0, xb .punpcklqdq .xmm0 .xmm3, xb .punpckhqdq .xmm1 .xmm3, + .movdquStore (at_ .rdx 0) .xmm0, .movdquStore (at_ .rdx 16) .xmm1, .alu .add .rdx (.imm 32)] ++ + [.alu .sub .rcx (.imm 1)] = pre2 o dz ++ (bf ++ post2) by simp [List.append_assoc]] + exact WP.mono (vstep2 hbf hblk hi o dz zi (hk i hi) (hsel i hi) hb'.consts hdx' h8' hS' hT' hwf' hw') + fun u' ⟨hS'', hdx'', h8'', hcx, hzf, hb''⟩ => ⟨⟨hS'', hdx'', by rw [h8'', h8', hstep i hi], + hb'.trans hb''⟩, hcx, hzf⟩ + +/-! ## The layer with `len = 1` -/ + +/-- The loads, the zetas and the gathering of the coefficients. -/ +abbrev pre1 (o : BitVec 8) (dz : BitVec 32) : List Instr := + [.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm2 (at_ .rdx 16)] ++ vzeta o ++ + [.alu .add .r8 (.imm dz), .xop (.pshufd .xmm0 .xmm0 0xD8), .xop (.pshufd .xmm2 .xmm2 0xD8), + xmov .xmm1 .xmm0, xb .punpcklqdq .xmm0 .xmm2, xb .punpckhqdq .xmm1 .xmm2] + +/-- The interleaving back, the stores and the counts. -/ +abbrev post1 : List Instr := + [xmov .xmm1 .xmm0, xb .punpckldq .xmm0 .xmm3, xb .punpckhdq .xmm1 .xmm3, + .movdquStore (at_ .rdx 0) .xmm0, .movdquStore (at_ .rdx 16) .xmm1, .alu .add .rdx (.imm 32), + .alu .sub .rcx (.imm 1)] + +omit hbf in +/-- Each coefficient after the first `b` blocks of the layer with `len = 1`. -/ +theorem layF1_get (F : Poly) (zi : Nat → Nat) {b : Nat} (hb : b ≤ 128) {j : Nat} (hj : j < 256) : + (layF blk F 1 zi b)[j]! = if j < 2 * b then + (if j % 2 = 0 then (op F[j]! F[j + 1]! (zetas (zi (j / 2)))).1 + else (op F[j - 1]! F[j]! (zetas (zi (j / 2)))).2) else F[j]! := by + induction b generalizing j with + | zero => rw [ite_eq_right (by omega)]; rfl + | succ b ih => + rw [layF, foldl_range_succ, ← layF, + hblk.get _ 1 _ _ 1 (by decide) (by decide) (by rw [n_eq]; omega) j (by rw [n_eq]; exact hj)] + by_cases h1 : 2 * 1 * b ≤ j ∧ j < 2 * 1 * b + 1 + · rw [ite_eq_left h1, ih (by omega) hj, ih (by omega) (by omega), show j / 2 = b by omega] + simp (disch := omega) only [ite_eq_left, ite_eq_right] + · rw [ite_eq_right h1] + by_cases h2 : 2 * 1 * b + 1 ≤ j ∧ j < 2 * 1 * b + 1 + 1 + · rw [ite_eq_left h2, ih (by omega) (by omega), ih (by omega) hj, show j / 2 = b by omega] + simp (disch := omega) only [ite_eq_left, ite_eq_right] + · rw [ite_eq_right h2, ih (by omega) hj] + by_cases h3 : j < 2 * b <;> simp (disch := omega) only [ite_eq_left, ite_eq_right] + +theorem vstep1 {fP sP : Addr} {i kz : Nat} (hi : i < 32) (o : BitVec 8) (dz : BitVec 32) (zi : Nat → Nat) + (hk : kz + 4 ≤ 256) (hsel : ∀ e < 4, kz + sel o e = zi (4 * i + e)) + {F : Poly} {s : State} (hc : VConsts s) (hdx : s.gpr .rdx = coeffAddr fP (8 * i)) + (h8 : s.gpr .r8 = coeffAddr sP kz) (hS : PolyIs s.mem fP (layF blk F 1 zi (4 * i))) + (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (.block (pre1 o dz ++ (bf ++ post1))) s fun s' => + PolyIs s'.mem fP (layF blk F 1 zi (4 * (i + 1))) ∧ s'.gpr .rdx = coeffAddr fP (8 * (i + 1)) ∧ + s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ + s'.zf = some (s.gpr .rcx - 1 == 0) ∧ BInv fP s s' := by + have j0 : 8 * i + 4 ≤ 256 := by omega + have j1 : 8 * i + 4 + 4 ≤ 256 := by omega + have a1 : coeffAddr fP (8 * i) + BitVec.ofNat 64 16 = coeffAddr fP (8 * i + 4) := coeffAddr_add _ _ 4 + have r0 := f_in (List.mem_append_right s.rd hwf) j0 + have r1 := f_in (List.mem_append_right s.rd hwf) j1 + have hk' : ∀ j < 4, kz + sel o j < 256 := fun j _ => by have := sel_lt o j; omega + generalize hG : layF blk F 1 zi (4 * i) = G at hS + have lx := dlanes_load hS j0 + have ly := dlanes_load hS j1 + rw [WP.block_append_iff, show pre1 o dz = [.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm2 (at_ .rdx 16)] ++ + (vzeta o ++ [.alu .add .r8 (.imm dz), .xop (.pshufd .xmm0 .xmm0 0xD8), .xop (.pshufd .xmm2 .xmm2 0xD8), + xmov .xmm1 .xmm0, xb .punpcklqdq .xmm0 .xmm2, xb .punpckhqdq .xmm1 .xmm2]) by simp, WP.block_append_iff] + vrund [hdx, a1, r0, r1] + rw [WP.block_append_iff] + refine WP.mono (vzeta_ok o hk' (by simp only [RegUpd.gpr_setXmm]; exact h8) + (by simp only [RegUpd.rd_setXmm, RegUpd.wr_setXmm]; exact tab_in (List.mem_append_right _ hw) hk) + (by simp only [RegUpd.mem_setXmm]; exact hT)) fun s1 ⟨z1, zo1, o1⟩ => ?_ + refine WP.mono (Q := fun (s1' : State) => DLanes (s1'.xmm .xmm0) (fun e => G[8 * i + 2 * e]!) ∧ + DLanes (s1'.xmm .xmm1) (fun e => G[8 * i + 2 * e + 1]!) ∧ + ZLanes (s1'.xmm .xmm13) (fun e => zetas (zi (4 * i + e))) ∧ ZOdd (s1'.xmm .xmm13) (s1'.xmm .xmm12) ∧ + VConsts s1' ∧ s1'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ GKeep [.r8] s s1') ?_ + fun s1' ⟨l0, l1, l13, l12, c1, h81, o1'⟩ => ?_ + · have g1 : s1.gpr = s.gpr := by rw [o1.gpr]; rfl + have m1 : s1.mem = s.mem := by rw [o1.mem]; rfl + have e1 : s1.rd = s.rd ∧ s1.wr = s.wr := ⟨by rw [o1.rd]; rfl, by rw [o1.wr]; rfl⟩ + have x1 : s1.mxcsr = s.mxcsr := by rw [o1.mxcsr]; rfl + have x0 : s1.xmm .xmm0 = s.mem.readW (coeffAddr fP (8 * i)) 128 := by + rw [o1.xmm _ (by decide)]; simp only [xmm_setXmm]; rfl + have x2 : s1.xmm .xmm2 = s.mem.readW (coeffAddr fP (8 * i + 4)) 128 := by + rw [o1.xmm _ (by decide)]; simp only [xmm_setXmm]; rfl + have c1 : VConsts s1 := xonly_vconsts o1 ((hc.setXmm (by decide) (by decide) _).setXmm (by decide) + (by decide) _) (by decide) (by decide) + vrund [g1, m1, e1.1, e1.2, x1, eval_movdqa] + refine ⟨?_, ?_, ?_, ?_, ⟨?_, ?_⟩, ⟨⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_⟩⟩ + · intro e he + rw [dword_punpcklqdq _ _ he, x0, x2] + split + · rw [dword_d8 _ he, ite_eq_left (by omega), lx _ (by omega)] + · rw [dword_d8 _ (by omega), ite_eq_left (by omega), ly _ (by omega)]; dsimp only + rw [show 8 * i + 4 + 2 * (e - 2) = 8 * i + 2 * e by omega] + · intro e he + rw [dword_punpckhqdq _ _ he, x0, x2] + split + · rw [dword_d8 _ (by omega), ite_eq_right (by omega), lx _ (by omega)]; dsimp only + rw [show 8 * i + (2 * (2 + e - 2) + 1) = 8 * i + 2 * e + 1 by omega] + · rw [dword_d8 _ he, ite_eq_right (by omega), ly _ (by omega)]; dsimp only + rw [show 8 * i + 4 + (2 * (e - 2) + 1) = 8 * i + 2 * e + 1 by omega] + · intro e he + rw [z1 e he]; dsimp only; rw [hsel e he] + · exact zo1 + · exact c1.q + · exact c1.qinv + · simp only [RegUpd.gpr_setXmm, RegUpd.gpr_setReg, RegUpd.gpr_setFlags, g1] + rw [ifn (by simpa using hr)] + all_goals simp only [RegUpd.rd_setXmm, RegUpd.wr_setXmm, RegUpd.mem_setXmm, mxcsr_setXmm, + RegUpd.rd_setReg, RegUpd.wr_setReg, RegUpd.mem_setReg, RegUpd.mxcsr_setReg, RegUpd.rd_setFlags, + RegUpd.wr_setFlags, RegUpd.mem_setFlags, RegUpd.mxcsr_setFlags, e1.1, e1.2, m1, x1] + rw [WP.block_append_iff] + refine WP.mono (hbf _ c1 _ _ _ l0 l1 l13 l12) fun s2 ⟨a0, a3, o2⟩ => ?_ + have g2 : s2.gpr .rdx = s.gpr .rdx := by rw [o2.gpr, o1'.keep.gpr (by decide)] + have e2 : s2.rd = s.rd ∧ s2.wr = s.wr := ⟨by rw [o2.rd, o1'.keep.2.1], by rw [o2.wr, o1'.keep.2.2]⟩ + have m2 : s2.mem = s.mem := by rw [o2.mem, o1'.mem] + have w0 := f_in hwf j0 + have w1 := f_in hwf j1 + have c2 := xonly_vconsts o2 c1 (by decide) (by decide) + simp only [post1, xmov, xb] + vrund [g2, e2.1, e2.2, m2, hdx, a1, w0, w1, sx32] + have r81 : s2.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz := by rw [o2.gpr, h81] + have rc1 : s2.gpr .rcx = s.gpr .rcx := by rw [o2.gpr, o1'.keep.gpr (by decide)] + have lg : ∀ b, b ≤ 128 → ∀ j, j < 256 → _ := fun b hb j hj => layF1_get hblk F zi (b := b) hb (j := j) hj + refine ⟨?_, by rw [show (32 : BitVec 64) = BitVec.ofNat 64 (4 * 8) from rfl, coeffAddr_add, Nat.mul_succ], r81, + by rw [rc1], by rw [rc1], ?_⟩ + · -- the coefficients stored + refine polyIs_write2 hS j0 j1 (by omega) + (a := fun e => (layF blk F 1 zi (4 * (i + 1)))[8 * i + e]!) + (b := fun e => (layF blk F 1 zi (4 * (i + 1)))[8 * i + 4 + e]!) + (fun e he => ?_) (fun e he => ?_) (fun j hj => ?_) + · rw [dword_punpckldq' _ _ he] + split + · rw [a0 (e / 2) (by omega)]; dsimp only + rw [show 8 * i + 2 * (e / 2) = 8 * i + e by omega, + show 4 * i + e / 2 = (8 * i + e) / 2 by omega, ← hG] + simp (disch := omega) only [lg, ite_eq_left, ite_eq_right] + · rw [a3 (e / 2) (by omega)]; dsimp only + rw [show 8 * i + 2 * (e / 2) = 8 * i + e - 1 by omega, show 8 * i + e - 1 + 1 = 8 * i + e by omega, + show 4 * i + e / 2 = (8 * i + e) / 2 by omega, ← hG] + simp (disch := omega) only [lg, ite_eq_left, ite_eq_right] + · rw [dword_punpckhdq' _ _ he, eval_movdqa] + split + · rw [a0 (2 + e / 2) (by omega)]; dsimp only + rw [show 8 * i + 2 * (2 + e / 2) = 8 * i + 4 + e by omega, + show 4 * i + (2 + e / 2) = (8 * i + 4 + e) / 2 by omega, ← hG] + simp (disch := omega) only [lg, ite_eq_left, ite_eq_right] + · rw [a3 (2 + e / 2) (by omega)]; dsimp only + rw [show 8 * i + 2 * (2 + e / 2) = 8 * i + 4 + e - 1 by omega, + show 8 * i + 4 + e - 1 + 1 = 8 * i + 4 + e by omega, + show 4 * i + (2 + e / 2) = (8 * i + 4 + e) / 2 by omega, ← hG] + simp (disch := omega) only [lg, ite_eq_left, ite_eq_right] + · rcases (by omega : (8 * i ≤ j ∧ j < 8 * i + 4) ∨ (8 * i + 4 ≤ j ∧ j < 8 * i + 8) ∨ + j < 8 * i ∨ 8 * i + 8 ≤ j) with h | h | h | h + · rw [ite_eq_left h, show 8 * i + (j - 8 * i) = j by omega] + · rw [ite_eq_right (by omega), ite_eq_left h, show 8 * i + 4 + (j - (8 * i + 4)) = j by omega] + · rw [ite_eq_right (by omega), ite_eq_right (by omega), ← hG] + simp (disch := omega) only [lg, ite_eq_left] + · rw [ite_eq_right (by omega), ite_eq_right (by omega), ← hG] + simp (disch := omega) only [lg, ite_eq_left, ite_eq_right] + · -- what the step keeps + refine ⟨⟨fun r hr => ?_, by simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags], + by simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags]⟩, frame_write2 (Frame.refl _ _) j0 j1 _ _, ⟨?_, ?_⟩, + by simp only [RegUpd.mxcsr_setReg, RegUpd.mxcsr_setFlags]; rw [o2.mxcsr, o1'.mxcsr]⟩ + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false] + rw [o2.gpr, o1'.keep.gpr (by simp [hr])] + · simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags, xmm_setXmm, reduceCtorEq, ite_false] + exact c2.q + · simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags, xmm_setXmm, reduceCtorEq, ite_false] + exact c2.qinv + +theorem vlay1_ok {fP sP : Addr} (k : Nat) (o : BitVec 8) (dz : BitVec 32) (zi kz : Nat → Nat) (hkz0 : kz 0 = k) + (hk : ∀ i < 32, kz i + 4 ≤ 256) (hsel : ∀ i < 32, ∀ e < 4, kz i + sel o e = zi (4 * i + e)) + (hstep : ∀ i < 32, coeffAddr sP (kz i) + BitVec.signExtend 64 dz = coeffAddr sP (kz (i + 1))) + {F : Poly} {s : State} (hc : VConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) + (hd : (pR sP).Disjoint (pR fP)) : + WP isa (vlay1 bf k o dz) s fun s' => PolyIs s'.mem fP (layF blk F 1 zi 128) ∧ BInv fP s s' := by + have hk0 : k + 4 ≤ 256 := by have := hk 0 (by decide); omega + refine WP.seq (WP.mono (vpre21 k hk0 hdi hsi) fun w ⟨hdx, h8, og⟩ => ?_) + refine WP.mono (wp_rcxLoop (N := 32) (by decide) (by decide) + (fun i u => PolyIs u.mem fP (layF blk F 1 zi (4 * i)) ∧ u.gpr .rdx = coeffAddr fP (8 * i) ∧ + u.gpr .r8 = coeffAddr sP (kz i) ∧ BInv fP w u) + (fun u ou _ => ⟨by rw [ou.mem, og.mem]; exact hS, + by rw [ou.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [ou.keep.gpr (by decide), h8, hkz0], ⟨ou.keep.mono (by simp), by rw [ou.mem]; exact Frame.refl _ _, + gonly_vconsts ou (gonly_vconsts og hc), ou.mxcsr⟩⟩) + (fun i hi u ⟨hS', hdx', h8', hb'⟩ => ?_)) fun u ⟨hS', _, _, hb'⟩ => + ⟨hS', ⟨(og.keep.trans hb'.keep).mono (by simp), by rw [← og.mem]; exact hb'.frame, hb'.consts, + by rw [hb'.mxcsr, og.mxcsr]⟩⟩ + have hT' : Tab zmTab u.mem sP 256 := (by rw [og.mem]; exact hT : Tab zmTab w.mem sP 256).frame hb'.frame + (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide) + have hwf' : pR fP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hwf + have hw' : pR sP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hw + rw [show [Instr.movdquLoad .xmm0 (at_ .rdx 0), .movdquLoad .xmm2 (at_ .rdx 16)] ++ vzeta o ++ + [.alu .add .r8 (.imm dz), .xop (.pshufd .xmm0 .xmm0 0xD8), .xop (.pshufd .xmm2 .xmm2 0xD8), + xmov .xmm1 .xmm0, xb .punpcklqdq .xmm0 .xmm2, xb .punpckhqdq .xmm1 .xmm2] ++ bf ++ + [xmov .xmm1 .xmm0, xb .punpckldq .xmm0 .xmm3, xb .punpckhdq .xmm1 .xmm3, + .movdquStore (at_ .rdx 0) .xmm0, .movdquStore (at_ .rdx 16) .xmm1, .alu .add .rdx (.imm 32)] ++ + [.alu .sub .rcx (.imm 1)] = pre1 o dz ++ (bf ++ post1) by simp [List.append_assoc]] + exact WP.mono (vstep1 hbf hblk hi o dz zi (hk i hi) (hsel i hi) hb'.consts hdx' h8' hS' hT' hwf' hw') + fun u' ⟨hS'', hdx'', h8'', hcx, hzf, hb''⟩ => ⟨⟨hS'', hdx'', by rw [h8'', h8', hstep i hi], + hb'.trans hb''⟩, hcx, hzf⟩ + +end + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VMem.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VMem.lean new file mode 100644 index 000000000..aff2c0409 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/VMem.lean @@ -0,0 +1,104 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLanes +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Table +import VerifiedGarbage.Proof.MlKem.X86_64.VMem +import VerifiedGarbage.Proof.MlDsa.Arith.Ntt + +/-! +# ML-DSA on x86-64: four coefficients at a time in memory + +Untrusted: everything here is checked by Lean. 16-byte loads of four +coefficients of a stored polynomial (`dlanes_load`) and stores of them +(`polyIs_write2`), and the table of the zetas in Montgomery form +(`Tab zmTab`), from which `vzeta` loads the zetas of up to four blocks +(`vzeta_ok`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (XOnly xmm_setXmm ifp ifn sel sel_lt add_ofNat_zero) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs zetas) + +theorem dlanes_load {m : Mem} {p : Addr} {F : Poly} (h : PolyIs m p F) {j : Nat} (hj : j + 4 ≤ 256) : + DLanes (m.readW (coeffAddr p j) 128) (fun e => F[j + e]!) := fun e he => by + rw [dword_readW _ _ he, coeffAddr_add, ← coeffAt_eq] + exact polyIs_toNat h (by rw [n_eq]; omega) + +/-- Coefficient `i` after storing `x` at coefficient `j`. -/ +theorem coeffAt_write128 (m : Mem) (p : Addr) {j : Nat} (hj : j + 4 ≤ 256) (x : BitVec 128) {i : Nat} + (hi : i < 256) : + coeffAt (m.writeW (coeffAddr p j) x) p i = if j ≤ i ∧ i < j + 4 then dword x (i - j) else coeffAt m p i := by + split + · rename_i h + rw [coeffAt_eq, show coeffAddr p i = coeffAddr p j + BitVec.ofNat 64 (4 * (i - j)) by + rw [coeffAddr_add, show j + (i - j) = i by omega]] + exact readW_writeW128 _ _ _ (by omega) + · exact Mem.readW_writeW_sep (Offset.sep p (by omega) (by omega) (by omega)) (by decide) + +/-- Two vectors stored into a polynomial, with the lanes `a` and `b`. -/ +theorem polyIs_write2 {m : Mem} {p : Addr} {P R : Poly} (hP : PolyIs m p P) {j j' : Nat} + (hj : j + 4 ≤ 256) (hj' : j' + 4 ≤ 256) (hsep : j + 4 ≤ j' ∨ j' + 4 ≤ j) {x y : BitVec 128} + {a b : Nat → Zq} (hx : DLanes x a) (hy : DLanes y b) + (hR : ∀ i < 256, R[i]! = if j ≤ i ∧ i < j + 4 then a (i - j) + else if j' ≤ i ∧ i < j' + 4 then b (i - j') else P[i]!) : + PolyIs ((m.writeW (coeffAddr p j) x).writeW (coeffAddr p j') y) p R := polyIs_of_toNat fun i hi => by + rw [n_eq] at hi + rw [coeffAt_write128 _ _ hj' _ hi, coeffAt_write128 _ _ hj _ hi, hR i hi] + by_cases h1 : j' ≤ i ∧ i < j' + 4 + · rw [ite_eq_left_of_eq_true _ _ (eq_true h1), ite_eq_right_of_eq_false _ _ (eq_false (by omega)), + ite_eq_left_of_eq_true _ _ (eq_true h1)] + exact hy _ (by omega) + · rw [ite_eq_right_of_eq_false _ _ (eq_false h1)] + by_cases h2 : j ≤ i ∧ i < j + 4 + · rw [ite_eq_left_of_eq_true _ _ (eq_true h2), ite_eq_left_of_eq_true _ _ (eq_true h2)] + exact hx _ (by omega) + · rw [ite_eq_right_of_eq_false _ _ (eq_false h2), ite_eq_right_of_eq_false _ _ (eq_false h2), + ite_eq_right_of_eq_false _ _ (eq_false h1)] + exact polyIs_toNat hP (by rw [n_eq]; exact hi) + +theorem pR_contains (p : Addr) {j : Nat} (hj : j + 4 ≤ 256) : (pR p).Contains (coeffAddr p j) 16 := + Offset.contains_base p (by omega) (by omega) + +theorem frame_write2 {m m' : Mem} {p : Addr} (hf : Frame [pR p] m m') {j j' : Nat} (hj : j + 4 ≤ 256) + (hj' : j' + 4 ≤ 256) (x y : BitVec 128) : + Frame [pR p] m ((m'.writeW (coeffAddr p j) x).writeW (coeffAddr p j') y) := + (hf.writeW (List.mem_singleton_self _) x (pR_contains p hj)).writeW (List.mem_singleton_self _) y + (pR_contains p hj') + +/-! ## The table of zetas -/ + +theorem zmTab_lt (k : Nat) : zmTab k < q := Nat.mod_lt _ (by decide) + +theorem zmTab_eq (k : Nat) : zmTab k = (zetas k).val * 2 ^ 32 % q := by + rw [zmTab, ← zetaNat_eq, zetaNat, Nat.mod_mul_mod] + +/-- The zeta at index `k` of the table. -/ +theorem tab_zeta {m : Mem} {zP : Addr} (ht : Tab zmTab m zP 256) {k : Nat} (hk : k < 256) : + (m.readW (coeffAddr zP k) 32).toNat = (zetas k).val * 2 ^ 32 % q := by + rw [← coeffAt_eq, ht k hk, BitVec.toNat_ofNat, Nat.mod_eq_of_lt (Nat.lt_trans (zmTab_lt k) (by decide)), + zmTab_eq] + +theorem dword_shufDwords_sel (a : BitVec 128) (o : BitVec 8) {i : Nat} (hi : i < 4) : + dword (shufDwords a o) i = dword a (sel o i) := dword_shufDwords a o hi + +theorem vzeta_ok (o : BitVec 8) {zP : Addr} {k : Nat} (hk : ∀ j < 4, k + sel o j < 256) {s : State} + (h8 : s.gpr .r8 = coeffAddr zP k) (hin : InRegions (s.rd ++ s.wr) (coeffAddr zP k) 16) + (ht : Tab zmTab s.mem zP 256) : + WP isa (.block (vzeta o)) s fun s' => + ZLanes (s'.xmm .xmm13) (fun i => zetas (k + sel o i)) ∧ ZOdd (s'.xmm .xmm13) (s'.xmm .xmm12) ∧ + XOnly [.xmm13, .xmm12] s s' := by + simp only [vzeta] + apply WP.of_runBlock + simp only [runBlock_cons, runStep_some, runBlock_nil, exec, XOp.exec, State.load128, ea_atD, + add_ofNat_zero, h8, hin, ite_true, Option.map_some, Option.some.injEq, exists_eq_left'] + refine ⟨fun i hi => ?_, fun j hj => ?_, by xonly⟩ + · simp only [xmm_setXmm, ite_true, ite_false, reduceCtorEq] + have hs := sel_lt o i + rw [dword_shufDwords_sel _ _ hi, dword_readW _ _ hs, coeffAddr_add] + exact tab_zeta ht (hk i hi) + · simp only [xmm_setXmm, ite_true, ite_false, reduceCtorEq] + rw [dword_shufDwords _ _ (by omega)] + rcases (by omega : j = 0 ∨ j = 1) with rfl | rfl <;> rfl + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Correct.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Correct.lean index 115c6bc54..6b95ef645 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Correct.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Correct.lean @@ -1,4 +1,5 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.PhaseO +import VerifiedGarbage.Proof.Framework.X86_64.Mxcsr /-! # ML-DSA signing on x86-64: correctness @@ -153,7 +154,7 @@ theorem entry_st {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} (h3 : Ok3 entry_bytes hf d6 (by omega) (ht.regs (.r13, .rdx) (by decide))⟩ theorem sign_correct {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} (h3 : Ok3 p) - (hmx : (Impl.MlDsa.X86_64.Sign.sign P p).allInstrs (fun i => !loadsMxcsr i) = true) (σ : State) + (hmx : ctlOk (Impl.MlDsa.X86_64.Sign.sign P p) = true) (σ : State) (hpre : (signK p D).pre σ) : ∃ t s', Exec isa (Impl.MlDsa.X86_64.Sign.sign P p) σ t s' ∧ abiPreserved σ s' ∧ (signK p D).post σ s' := by have hc := allChk_ok h3 @@ -182,7 +183,7 @@ theorem sign_correct {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} (h3 : ⟨hg, s₄, h₄, hr, hm⟩ obtain ⟨t, s', he, hF⟩ := main obtain ⟨hg, s₄, h₄, hr, hm⟩ := hF - refine ⟨t, s', he, abiPreserved_of_exec hmx he hg, ?_⟩ + refine ⟨t, s', he, abiPreserved_of_ctl hmx he hg, ?_⟩ have e14 : pa s₄ (.r14, 0) = σ.gpr .rcx := by rw [pa, h₄.st.top.regs (.r14, .rcx) (by decide), VG.Proof.MlKem.X86_64.add_ofNat_zero] show Outcome _ _ _ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean index de1fbb3cd..089924f6d 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean @@ -63,7 +63,7 @@ theorem signK_implies {p : Params} (h3 : Ok3 p) : · sig_implies_sat [signContractT, signSig, X86_64.abi, X86_64.argRegs] [signSat] using signSat mlDsa87 theorem sign_verified {p : Params} (h3 : Ok3 p) - (hmx : (Impl.MlDsa.X86_64.Sign.sign prims p).allInstrs (fun i => !loadsMxcsr i) = true) : + (hmx : ctlOk (Impl.MlDsa.X86_64.Sign.sign prims p) = true) : Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign prims p) (signContractT p X86_64.abi signStack) := Verified.of_correct (sign_correct prims_ok h3 hmx) (sign_ct prims_ok h3) (signK_implies h3) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Correct.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Correct.lean index 934c283a8..94176d623 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Correct.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Correct.lean @@ -128,6 +128,6 @@ theorem verify_correct {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ para rcases hr with ⟨e, hb⟩ | ⟨e, hb⟩ · exact .inl ⟨by rw [hres, e]; rfl, hb⟩ · exact .inr ⟨by rw [hres, e]; rfl, hb⟩⟩ : gprPreserved σ s₃ ∧ (verifyK p).post σ s₃))) - exact ⟨t, s', he, abiPreserved_of_exec (verify_mxcsr C hp) he hF.1, hF.2⟩ + exact ⟨t, s', he, abiPreserved_of_ctl (verify_ctl C hp) he hF.1, hF.2⟩ end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean index e00400b78..430f0bb51 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean @@ -1,6 +1,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Verify.Call import VerifiedGarbage.Proof.MlKem.X86_64.KCall import VerifiedGarbage.Proof.Framework.X86_64.Abi +import VerifiedGarbage.Proof.Framework.X86_64.Mxcsr /-! # ML-DSA verification on x86-64: entry to a callee @@ -25,14 +26,14 @@ open VG.Spec.Sha3 (bytesAt) /-- A callee: correct and constant time under the contract `k` (a shared contract with 16 bytes of stack), not writing `rsp`, calling at most two -deep, and never loading MXCSR or writing the stack pointer (which its -callers' artifacts check). -/ +deep, loading MXCSR only to restore it (`ctlOk`), and never writing the +stack pointer (which its callers' artifacts check). -/ structure CalleeOk (c : Prog isa) (k : Contract isa) : Prop where correct : ∀ s, k.pre s → ∃ t s', Exec isa c s t s' ∧ abiPreserved s s' ∧ k.post s s' ct : ConstantTime isa k.pre k.pub c nosp : NoSp c depth : c.depth ≤ 2 - mxcsr : c.allInstrs (fun i => !loadsMxcsr i) = true + ctl : ctlOk c = true spSafe : c.all (fun i => !isa.writesSp i) = true /-- A callee verified against its shared contract with at most 16 bytes of stack. -/ @@ -40,7 +41,7 @@ theorem CalleeOk.of_verified {c : Prog isa} {sig : Sig} {pre : Curry (sig.words {post : sig.Post X86_64.abi.ptrBits} {wa : Bool} {leak : Option (Curry (sig.words X86_64.abi.ptrBits) (Mem → List Nat))} {n : Nat} (h : Verified X86_64.target c (sig.contract X86_64.abi pre post wa n leak)) (hn : n ≤ 16) - (hsp : NoSp c) (hd : c.depth ≤ 2) (hmx : c.allInstrs (fun i => !loadsMxcsr i) = true) + (hsp : NoSp c) (hd : c.depth ≤ 2) (hmx : ctlOk c = true) (hss : c.all (fun i => !isa.writesSp i) = true) : CalleeOk c (sig.contract X86_64.abi pre post wa 16 leak) := ⟨fun s hs => h.1 s (pre_stack hn hs), diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean index 090c626e4..ce8c82e43 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean @@ -8,8 +8,9 @@ Untrusted: everything here is checked by Lean. A property `q` of every instruction of `verify P p` (`Code.allInstrs q`) holds if it holds of every instruction of the primitives `P` and of `verify P0 p`, the same code with the primitives empty (`verify_q`), which the kernel evaluates. So it never -loads MXCSR (`verify_mxcsr`) or writes the stack pointer (`verify_spSafe`) -if the primitives do not. +writes the stack pointer (`verify_spSafe`) if the primitives do not. Likewise +for `ctlC` (`verify_c`): it loads MXCSR only to restore it (`verify_ctl`) if +the primitives do (`ctlOk`). -/ namespace VG.Proof.MlDsa.X86_64.Verify @@ -107,19 +108,87 @@ theorem Code.allInstrs_of_all {I C : Type} {q : I → Bool} {c : Code I C} (h : | block is => induction is <;> simp_all [Code.all, Code.allInstrs] | _ => simp_all [Code.all, Code.allInstrs] -theorem verify0_mxcsr : ∀ p ∈ params, (verify P0 p).allInstrs (fun i => !loadsMxcsr i) = true := by +theorem verify0_sp : ∀ p ∈ params, (verify P0 p).allInstrs (fun i => !isa.writesSp i) = true := by decide +kernel -theorem verify0_sp : ∀ p ∈ params, (verify P0 p).allInstrs (fun i => !isa.writesSp i) = true := by +/-! ## MXCSR -/ + +/-- Every primitive of `P` loads MXCSR only to restore it. -/ +structure PrimsC (P : Prims) : Prop where + ntt : ctlOk P.ntt = true + invNtt : ctlOk P.invNtt = true + mul : ctlOk P.mul = true + mulAdd : ctlOk P.mulAdd = true + sub : ctlOk P.sub = true + rejNtt : ctlOk P.rejNtt = true + ball : ctlOk P.ball = true + useHint : ctlOk P.useHint = true + simpleBitPack : ctlOk P.simpleBitPack = true + bitUnpack : ctlOk P.bitUnpack = true + unpackT1 : ctlOk P.unpackT1 = true + hintUnpack : ctlOk P.hintUnpack = true + normLt : ctlOk P.normLt = true + +/-- `ctlC` holds of `c` exactly when it does of `c'`. -/ +def SameC (c c' : Prog isa) : Prop := ctlC c = ctlC c' + +theorem SameC.seq {a a' b b' : Prog isa} (ha : SameC a a') (hb : SameC b b') : SameC (.seq a b) (.seq a' b') := by + show (ctlC a && ctlC b) = (ctlC a' && ctlC b') + rw [show ctlC a = ctlC a' from ha, show ctlC b = ctlC b' from hb] + +theorem SameC.call {c : Prog isa} (hc : ctlOk c = true) (n : String) (as : List (Reg × Arg)) : + SameC (callAt n c as) (callAt n (.block []) as) := by + show (_ && ctlOk c) = (_ && true) + rw [hc] + +theorem SameC.seqR {f g : Nat → Prog isa} (h : ∀ k, SameC (f k) (g k)) : ∀ a n, SameC (seqR f a n) (seqR g a n) + | _, 0 => rfl + | a, n + 1 => (h a).seq (SameC.seqR h (a + 1) n) + +theorem SameC.ifOk {c c' : Prog isa} (h : SameC c c') : SameC (ifOk c) (ifOk c') := by + refine SameC.seq rfl ?_ + show (ctlC c && _) = (ctlC c' && _) + rw [show ctlC c = ctlC c' from h] + +theorem SameC.sampled {c c' : Prog isa} (h : SameC c c') (a : Ptr) : SameC (sampled c a) (sampled c' a) := + h.seq rfl + +section +variable {P : Prims} (hP : PrimsC P) (p : Params) +include hP + +theorem verify_c : SameC (verify P p) (verify P0 p) := by + have aOne : ∀ e, SameC (aOne P e) (aOne P0 e) := fun e => + SameC.seq rfl (SameC.sampled (SameC.call hP.rejNtt _ _) _) + have dot : ∀ r, SameC (dot P p r) (dot P0 p r) := fun r => + (SameC.call hP.mul _ _).seq (SameC.seqR (fun _ => SameC.call hP.mulAdd _ _) _ _) + have row : ∀ r, SameC (row P p r) (row P0 p r) := fun r => + (dot r).seq ((SameC.call hP.unpackT1 _ _).seq ((SameC.call hP.ntt _ _).seq ((SameC.call hP.mul _ _).seq + ((SameC.call hP.sub _ _).seq ((SameC.call hP.invNtt _ _).seq ((SameC.call hP.useHint _ _).seq + (SameC.call hP.simpleBitPack _ _))))))) + have samples : SameC (samples P p) (samples P0 p) := + SameC.seq rfl ((SameC.seqR (fun r => SameC.seqR aOne _ _) _ _).seq + (SameC.sampled (SameC.call hP.ball _ _) _)) + have compute : SameC (compute P p) (compute P0 p) := + (SameC.seqR (fun _ => SameC.call hP.ntt _ _) _ _).seq ((SameC.call hP.ntt _ _).seq + ((SameC.seqR row _ _).seq rfl)) + have zOne : ∀ i, SameC (zOne P p i) (zOne P0 p i) := fun _ => + (SameC.call hP.bitUnpack _ _).seq ((SameC.call hP.normLt _ _).seq rfl) + exact SameC.seq rfl ((((SameC.call hP.hintUnpack _ _).seq rfl).seq (SameC.ifOk ((SameC.seqR zOne _ _).seq + (SameC.ifOk (samples.seq compute))))).seq rfl) + +end + +theorem verify0_ctlC : ∀ p ∈ params, ctlC (verify P0 p) = true := by decide +kernel variable {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) include C hp -theorem verify_mxcsr : (verify P p).allInstrs (fun i => !loadsMxcsr i) = true := - (verify_q ⟨C.ntt.mxcsr, C.invNtt.mxcsr, C.mul.mxcsr, C.mulAdd.mxcsr, C.sub.mxcsr, C.rejNtt.mxcsr, C.ball.mxcsr, - C.useHint.mxcsr, C.simpleBitPack.mxcsr, C.bitUnpack.mxcsr, C.unpackT1.mxcsr, C.hintUnpack.mxcsr, - C.normLt.mxcsr⟩ p).trans (verify0_mxcsr p hp) +theorem verify_ctl : ctlOk (verify P p) = true := + ctlOk_of_ctlC ((verify_c ⟨C.ntt.ctl, C.invNtt.ctl, C.mul.ctl, C.mulAdd.ctl, C.sub.ctl, C.rejNtt.ctl, C.ball.ctl, + C.useHint.ctl, C.simpleBitPack.ctl, C.bitUnpack.ctl, C.unpackT1.ctl, C.hintUnpack.ctl, + C.normLt.ctl⟩ p).trans (verify0_ctlC p hp)) theorem verify_spSafe : (verify P p).all (fun i => !isa.writesSp i) = true := Code.all_of_allInstrs ((verify_q ⟨Code.allInstrs_of_all C.ntt.spSafe, Code.allInstrs_of_all C.invNtt.spSafe, diff --git a/src/asm/x86_64/mldsa.rs b/src/asm/x86_64/mldsa.rs index 8b5809e4c..59b6ad833 100644 --- a/src/asm/x86_64/mldsa.rs +++ b/src/asm/x86_64/mldsa.rs @@ -6,7 +6,7 @@ /// /// Contract: `VG.Spec.MlDsa.nttContract`. Constant time: only the pointers may affect timing, not the data. /// -/// The function stores a table of the 256 zetas in `scratch`. +/// The function computes on four coefficients at a time in SSE2 registers, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. /// /// # Safety /// @@ -19,850 +19,691 @@ #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa_ntt(f: *mut [u32; 256], scratch: *mut [u64; 128]) { core::arch::naked_asm!( - "mov r9, rsi", - "mov eax, 1", - "mov DWORD PTR [r9], eax", - "mov eax, 4808194", - "mov DWORD PTR [r9+4], eax", - "mov eax, 3765607", - "mov DWORD PTR [r9+8], eax", - "mov eax, 3761513", - "mov DWORD PTR [r9+12], eax", - "mov eax, 5178923", - "mov DWORD PTR [r9+16], eax", - "mov eax, 5496691", - "mov DWORD PTR [r9+20], eax", - "mov eax, 5234739", - "mov DWORD PTR [r9+24], eax", - "mov eax, 5178987", - "mov DWORD PTR [r9+28], eax", - "mov eax, 7778734", - "mov DWORD PTR [r9+32], eax", - "mov eax, 3542485", - "mov DWORD PTR [r9+36], eax", - "mov eax, 2682288", - "mov DWORD PTR [r9+40], eax", - "mov eax, 2129892", - "mov DWORD PTR [r9+44], eax", - "mov eax, 3764867", - "mov DWORD PTR [r9+48], eax", - "mov eax, 7375178", - "mov DWORD PTR [r9+52], eax", - "mov eax, 557458", - "mov DWORD PTR [r9+56], eax", - "mov eax, 7159240", - "mov DWORD PTR [r9+60], eax", - "mov eax, 5010068", - "mov DWORD PTR [r9+64], eax", - "mov eax, 4317364", - "mov DWORD PTR [r9+68], eax", - "mov eax, 2663378", - "mov DWORD PTR [r9+72], eax", - "mov eax, 6705802", - "mov DWORD PTR [r9+76], eax", - "mov eax, 4855975", - "mov DWORD PTR [r9+80], eax", - "mov eax, 7946292", - "mov DWORD PTR [r9+84], eax", - "mov eax, 676590", - "mov DWORD PTR [r9+88], eax", - "mov eax, 7044481", - "mov DWORD PTR [r9+92], eax", - "mov eax, 5152541", - "mov DWORD PTR [r9+96], eax", - "mov eax, 1714295", - "mov DWORD PTR [r9+100], eax", - "mov eax, 2453983", - "mov DWORD PTR [r9+104], eax", - "mov eax, 1460718", - "mov DWORD PTR [r9+108], eax", - "mov eax, 7737789", - "mov DWORD PTR [r9+112], eax", - "mov eax, 4795319", - "mov DWORD PTR [r9+116], eax", - "mov eax, 2815639", - "mov DWORD PTR [r9+120], eax", - "mov eax, 2283733", - "mov DWORD PTR [r9+124], eax", - "mov eax, 3602218", - "mov DWORD PTR [r9+128], eax", - "mov eax, 3182878", - "mov DWORD PTR [r9+132], eax", - "mov eax, 2740543", - "mov DWORD PTR [r9+136], eax", - "mov eax, 4793971", - "mov DWORD PTR [r9+140], eax", - "mov eax, 5269599", - "mov DWORD PTR [r9+144], eax", - "mov eax, 2101410", - "mov DWORD PTR [r9+148], eax", - "mov eax, 3704823", - "mov DWORD PTR [r9+152], eax", - "mov eax, 1159875", - "mov DWORD PTR [r9+156], eax", - "mov eax, 394148", - "mov DWORD PTR [r9+160], eax", - "mov eax, 928749", - "mov DWORD PTR [r9+164], eax", - "mov eax, 1095468", - "mov DWORD PTR [r9+168], eax", - "mov eax, 4874037", - "mov DWORD PTR [r9+172], eax", - "mov eax, 2071829", - "mov DWORD PTR [r9+176], eax", - "mov eax, 4361428", - "mov DWORD PTR [r9+180], eax", - "mov eax, 3241972", - "mov DWORD PTR [r9+184], eax", - "mov eax, 2156050", - "mov DWORD PTR [r9+188], eax", - "mov eax, 3415069", - "mov DWORD PTR [r9+192], eax", - "mov eax, 1759347", - "mov DWORD PTR [r9+196], eax", - "mov eax, 7562881", - "mov DWORD PTR [r9+200], eax", - "mov eax, 4805951", - "mov DWORD PTR [r9+204], eax", - "mov eax, 3756790", - "mov DWORD PTR [r9+208], eax", - "mov eax, 6444618", - "mov DWORD PTR [r9+212], eax", - "mov eax, 6663429", - "mov DWORD PTR [r9+216], eax", - "mov eax, 4430364", - "mov DWORD PTR [r9+220], eax", - "mov eax, 5483103", - "mov DWORD PTR [r9+224], eax", - "mov eax, 3192354", - "mov DWORD PTR [r9+228], eax", - "mov eax, 556856", - "mov DWORD PTR [r9+232], eax", - "mov eax, 3870317", - "mov DWORD PTR [r9+236], eax", - "mov eax, 2917338", - "mov DWORD PTR [r9+240], eax", - "mov eax, 1853806", - "mov DWORD PTR [r9+244], eax", - "mov eax, 3345963", - "mov DWORD PTR [r9+248], eax", - "mov eax, 1858416", - "mov DWORD PTR [r9+252], eax", - "mov eax, 3073009", - "mov DWORD PTR [r9+256], eax", - "mov eax, 1277625", - "mov DWORD PTR [r9+260], eax", - "mov eax, 5744944", - "mov DWORD PTR [r9+264], eax", - "mov eax, 3852015", - "mov DWORD PTR [r9+268], eax", - "mov eax, 4183372", - "mov DWORD PTR [r9+272], eax", - "mov eax, 5157610", - "mov DWORD PTR [r9+276], eax", - "mov eax, 5258977", - "mov DWORD PTR [r9+280], eax", - "mov eax, 8106357", - "mov DWORD PTR [r9+284], eax", - "mov eax, 2508980", - "mov DWORD PTR [r9+288], eax", - "mov eax, 2028118", - "mov DWORD PTR [r9+292], eax", - "mov eax, 1937570", - "mov DWORD PTR [r9+296], eax", - "mov eax, 4564692", - "mov DWORD PTR [r9+300], eax", - "mov eax, 2811291", - "mov DWORD PTR [r9+304], eax", - "mov eax, 5396636", - "mov DWORD PTR [r9+308], eax", - "mov eax, 7270901", - "mov DWORD PTR [r9+312], eax", - "mov eax, 4158088", - "mov DWORD PTR [r9+316], eax", - "mov eax, 1528066", - "mov DWORD PTR [r9+320], eax", - "mov eax, 482649", - "mov DWORD PTR [r9+324], eax", - "mov eax, 1148858", - "mov DWORD PTR [r9+328], eax", - "mov eax, 5418153", - "mov DWORD PTR [r9+332], eax", - "mov eax, 7814814", - "mov DWORD PTR [r9+336], eax", - "mov eax, 169688", - "mov DWORD PTR [r9+340], eax", - "mov eax, 2462444", - "mov DWORD PTR [r9+344], eax", - "mov eax, 5046034", - "mov DWORD PTR [r9+348], eax", - "mov eax, 4213992", - "mov DWORD PTR [r9+352], eax", - "mov eax, 4892034", - "mov DWORD PTR [r9+356], eax", - "mov eax, 1987814", - "mov DWORD PTR [r9+360], eax", - "mov eax, 5183169", - "mov DWORD PTR [r9+364], eax", - "mov eax, 1736313", - "mov DWORD PTR [r9+368], eax", - "mov eax, 235407", - "mov DWORD PTR [r9+372], eax", - "mov eax, 5130263", - "mov DWORD PTR [r9+376], eax", - "mov eax, 3258457", - "mov DWORD PTR [r9+380], eax", - "mov eax, 5801164", - "mov DWORD PTR [r9+384], eax", - "mov eax, 1787943", - "mov DWORD PTR [r9+388], eax", - "mov eax, 5989328", - "mov DWORD PTR [r9+392], eax", - "mov eax, 6125690", - "mov DWORD PTR [r9+396], eax", - "mov eax, 3482206", - "mov DWORD PTR [r9+400], eax", - "mov eax, 4197502", - "mov DWORD PTR [r9+404], eax", - "mov eax, 7080401", - "mov DWORD PTR [r9+408], eax", - "mov eax, 6018354", - "mov DWORD PTR [r9+412], eax", - "mov eax, 7062739", - "mov DWORD PTR [r9+416], eax", - "mov eax, 2461387", - "mov DWORD PTR [r9+420], eax", - "mov eax, 3035980", - "mov DWORD PTR [r9+424], eax", - "mov eax, 621164", - "mov DWORD PTR [r9+428], eax", - "mov eax, 3901472", - "mov DWORD PTR [r9+432], eax", - "mov eax, 7153756", - "mov DWORD PTR [r9+436], eax", - "mov eax, 2925816", - "mov DWORD PTR [r9+440], eax", - "mov eax, 3374250", - "mov DWORD PTR [r9+444], eax", - "mov eax, 1356448", - "mov DWORD PTR [r9+448], eax", - "mov eax, 5604662", - "mov DWORD PTR [r9+452], eax", - "mov eax, 2683270", - "mov DWORD PTR [r9+456], eax", - "mov eax, 5601629", - "mov DWORD PTR [r9+460], eax", - "mov eax, 4912752", - "mov DWORD PTR [r9+464], eax", - "mov eax, 2312838", - "mov DWORD PTR [r9+468], eax", - "mov eax, 7727142", - "mov DWORD PTR [r9+472], eax", - "mov eax, 7921254", - "mov DWORD PTR [r9+476], eax", - "mov eax, 348812", - "mov DWORD PTR [r9+480], eax", - "mov eax, 8052569", - "mov DWORD PTR [r9+484], eax", - "mov eax, 1011223", - "mov DWORD PTR [r9+488], eax", - "mov eax, 6026202", - "mov DWORD PTR [r9+492], eax", - "mov eax, 4561790", - "mov DWORD PTR [r9+496], eax", - "mov eax, 6458164", - "mov DWORD PTR [r9+500], eax", - "mov eax, 6143691", - "mov DWORD PTR [r9+504], eax", - "mov eax, 1744507", - "mov DWORD PTR [r9+508], eax", - "mov eax, 1753", - "mov DWORD PTR [r9+512], eax", - "mov eax, 6444997", - "mov DWORD PTR [r9+516], eax", - "mov eax, 5720892", - "mov DWORD PTR [r9+520], eax", - "mov eax, 6924527", - "mov DWORD PTR [r9+524], eax", - "mov eax, 2660408", - "mov DWORD PTR [r9+528], eax", - "mov eax, 6600190", - "mov DWORD PTR [r9+532], eax", - "mov eax, 8321269", - "mov DWORD PTR [r9+536], eax", - "mov eax, 2772600", - "mov DWORD PTR [r9+540], eax", - "mov eax, 1182243", - "mov DWORD PTR [r9+544], eax", - "mov eax, 87208", - "mov DWORD PTR [r9+548], eax", - "mov eax, 636927", - "mov DWORD PTR [r9+552], eax", - "mov eax, 4415111", - "mov DWORD PTR [r9+556], eax", - "mov eax, 4423672", - "mov DWORD PTR [r9+560], eax", - "mov eax, 6084020", - "mov DWORD PTR [r9+564], eax", - "mov eax, 5095502", - "mov DWORD PTR [r9+568], eax", - "mov eax, 4663471", - "mov DWORD PTR [r9+572], eax", - "mov eax, 8352605", - "mov DWORD PTR [r9+576], eax", - "mov eax, 822541", - "mov DWORD PTR [r9+580], eax", - "mov eax, 1009365", - "mov DWORD PTR [r9+584], eax", - "mov eax, 5926272", - "mov DWORD PTR [r9+588], eax", - "mov eax, 6400920", - "mov DWORD PTR [r9+592], eax", - "mov eax, 1596822", - "mov DWORD PTR [r9+596], eax", - "mov eax, 4423473", - "mov DWORD PTR [r9+600], eax", - "mov eax, 4620952", - "mov DWORD PTR [r9+604], eax", - "mov eax, 6695264", - "mov DWORD PTR [r9+608], eax", - "mov eax, 4969849", - "mov DWORD PTR [r9+612], eax", - "mov eax, 2678278", - "mov DWORD PTR [r9+616], eax", - "mov eax, 4611469", - "mov DWORD PTR [r9+620], eax", - "mov eax, 4829411", - "mov DWORD PTR [r9+624], eax", - "mov eax, 635956", - "mov DWORD PTR [r9+628], eax", - "mov eax, 8129971", - "mov DWORD PTR [r9+632], eax", - "mov eax, 5925040", - "mov DWORD PTR [r9+636], eax", - "mov eax, 4234153", - "mov DWORD PTR [r9+640], eax", - "mov eax, 6607829", - "mov DWORD PTR [r9+644], eax", - "mov eax, 2192938", - "mov DWORD PTR [r9+648], eax", - "mov eax, 6653329", - "mov DWORD PTR [r9+652], eax", - "mov eax, 2387513", - "mov DWORD PTR [r9+656], eax", - "mov eax, 4768667", - "mov DWORD PTR [r9+660], eax", - "mov eax, 8111961", - "mov DWORD PTR [r9+664], eax", - "mov eax, 5199961", - "mov DWORD PTR [r9+668], eax", - "mov eax, 3747250", - "mov DWORD PTR [r9+672], eax", - "mov eax, 2296099", - "mov DWORD PTR [r9+676], eax", - "mov eax, 1239911", - "mov DWORD PTR [r9+680], eax", - "mov eax, 4541938", - "mov DWORD PTR [r9+684], eax", - "mov eax, 3195676", - "mov DWORD PTR [r9+688], eax", - "mov eax, 2642980", - "mov DWORD PTR [r9+692], eax", - "mov eax, 1254190", - "mov DWORD PTR [r9+696], eax", - "mov eax, 8368000", - "mov DWORD PTR [r9+700], eax", - "mov eax, 2998219", - "mov DWORD PTR [r9+704], eax", - "mov eax, 141835", - "mov DWORD PTR [r9+708], eax", - "mov eax, 8291116", - "mov DWORD PTR [r9+712], eax", - "mov eax, 2513018", - "mov DWORD PTR [r9+716], eax", - "mov eax, 7025525", - "mov DWORD PTR [r9+720], eax", - "mov eax, 613238", - "mov DWORD PTR [r9+724], eax", - "mov eax, 7070156", - "mov DWORD PTR [r9+728], eax", - "mov eax, 6161950", - "mov DWORD PTR [r9+732], eax", - "mov eax, 7921677", - "mov DWORD PTR [r9+736], eax", - "mov eax, 6458423", - "mov DWORD PTR [r9+740], eax", - "mov eax, 4040196", - "mov DWORD PTR [r9+744], eax", - "mov eax, 4908348", - "mov DWORD PTR [r9+748], eax", - "mov eax, 2039144", - "mov DWORD PTR [r9+752], eax", - "mov eax, 6500539", - "mov DWORD PTR [r9+756], eax", - "mov eax, 7561656", - "mov DWORD PTR [r9+760], eax", - "mov eax, 6201452", - "mov DWORD PTR [r9+764], eax", - "mov eax, 6757063", - "mov DWORD PTR [r9+768], eax", - "mov eax, 2105286", - "mov DWORD PTR [r9+772], eax", - "mov eax, 6006015", - "mov DWORD PTR [r9+776], eax", - "mov eax, 6346610", - "mov DWORD PTR [r9+780], eax", - "mov eax, 586241", - "mov DWORD PTR [r9+784], eax", - "mov eax, 7200804", - "mov DWORD PTR [r9+788], eax", - "mov eax, 527981", - "mov DWORD PTR [r9+792], eax", - "mov eax, 5637006", - "mov DWORD PTR [r9+796], eax", - "mov eax, 6903432", - "mov DWORD PTR [r9+800], eax", - "mov eax, 1994046", - "mov DWORD PTR [r9+804], eax", - "mov eax, 2491325", - "mov DWORD PTR [r9+808], eax", - "mov eax, 6987258", - "mov DWORD PTR [r9+812], eax", - "mov eax, 507927", - "mov DWORD PTR [r9+816], eax", - "mov eax, 7192532", - "mov DWORD PTR [r9+820], eax", - "mov eax, 7655613", - "mov DWORD PTR [r9+824], eax", - "mov eax, 6545891", - "mov DWORD PTR [r9+828], eax", - "mov eax, 5346675", - "mov DWORD PTR [r9+832], eax", - "mov eax, 8041997", - "mov DWORD PTR [r9+836], eax", - "mov eax, 2647994", - "mov DWORD PTR [r9+840], eax", - "mov eax, 3009748", - "mov DWORD PTR [r9+844], eax", - "mov eax, 5767564", - "mov DWORD PTR [r9+848], eax", - "mov eax, 4148469", - "mov DWORD PTR [r9+852], eax", - "mov eax, 749577", - "mov DWORD PTR [r9+856], eax", - "mov eax, 4357667", - "mov DWORD PTR [r9+860], eax", - "mov eax, 3980599", - "mov DWORD PTR [r9+864], eax", - "mov eax, 2569011", - "mov DWORD PTR [r9+868], eax", - "mov eax, 6764887", - "mov DWORD PTR [r9+872], eax", - "mov eax, 1723229", - "mov DWORD PTR [r9+876], eax", - "mov eax, 1665318", - "mov DWORD PTR [r9+880], eax", - "mov eax, 2028038", - "mov DWORD PTR [r9+884], eax", - "mov eax, 1163598", - "mov DWORD PTR [r9+888], eax", - "mov eax, 5011144", - "mov DWORD PTR [r9+892], eax", - "mov eax, 3994671", - "mov DWORD PTR [r9+896], eax", - "mov eax, 8368538", - "mov DWORD PTR [r9+900], eax", - "mov eax, 7009900", - "mov DWORD PTR [r9+904], eax", - "mov eax, 3020393", - "mov DWORD PTR [r9+908], eax", - "mov eax, 3363542", - "mov DWORD PTR [r9+912], eax", - "mov eax, 214880", - "mov DWORD PTR [r9+916], eax", - "mov eax, 545376", - "mov DWORD PTR [r9+920], eax", - "mov eax, 7609976", - "mov DWORD PTR [r9+924], eax", - "mov eax, 3105558", - "mov DWORD PTR [r9+928], eax", - "mov eax, 7277073", - "mov DWORD PTR [r9+932], eax", - "mov eax, 508145", - "mov DWORD PTR [r9+936], eax", - "mov eax, 7826699", - "mov DWORD PTR [r9+940], eax", - "mov eax, 860144", - "mov DWORD PTR [r9+944], eax", - "mov eax, 3430436", - "mov DWORD PTR [r9+948], eax", - "mov eax, 140244", - "mov DWORD PTR [r9+952], eax", - "mov eax, 6866265", - "mov DWORD PTR [r9+956], eax", - "mov eax, 6195333", - "mov DWORD PTR [r9+960], eax", - "mov eax, 3123762", - "mov DWORD PTR [r9+964], eax", - "mov eax, 2358373", - "mov DWORD PTR [r9+968], eax", - "mov eax, 6187330", - "mov DWORD PTR [r9+972], eax", - "mov eax, 5365997", - "mov DWORD PTR [r9+976], eax", - "mov eax, 6663603", - "mov DWORD PTR [r9+980], eax", - "mov eax, 2926054", - "mov DWORD PTR [r9+984], eax", - "mov eax, 7987710", - "mov DWORD PTR [r9+988], eax", - "mov eax, 8077412", - "mov DWORD PTR [r9+992], eax", - "mov eax, 3531229", - "mov DWORD PTR [r9+996], eax", - "mov eax, 4405932", - "mov DWORD PTR [r9+1000], eax", - "mov eax, 4606686", - "mov DWORD PTR [r9+1004], eax", - "mov eax, 1900052", - "mov DWORD PTR [r9+1008], eax", - "mov eax, 7598542", - "mov DWORD PTR [r9+1012], eax", - "mov eax, 1054478", - "mov DWORD PTR [r9+1016], eax", - "mov eax, 7648983", - "mov DWORD PTR [r9+1020], eax", - "mov rsi, rdi", - "mov r8, r9", + "stmxcsr DWORD PTR [rsi+768]", + "mov r11d, DWORD PTR [rsi+768]", + "and r11d, 65535", + "mov eax, 8127", + "mov DWORD PTR [rsi+772], eax", + "ldmxcsr DWORD PTR [rsi+772]", + "lfence", + "movabs r9, 111012023893504", + "mov QWORD PTR [rsi], r9", + "movabs r9, 33764919763013891", + "mov QWORD PTR [rsi+8], r9", + "movabs r9, 32652304184483396", + "mov QWORD PTR [rsi+16], r9", + "movabs r9, 2003464812134697", + "mov QWORD PTR [rsi+24], r9", + "movabs r9, 10107995079564843", + "mov QWORD PTR [rsi+32], r9", + "movabs r9, 27008953388524718", + "mov QWORD PTR [rsi+40], r9", + "movabs r9, 23603259066260085", + "mov QWORD PTR [rsi+48], r9", + "movabs r9, 11510954738022985", + "mov QWORD PTR [rsi+56], r9", + "movabs r9, 4398527550166616", + "mov QWORD PTR [rsi+64], r9", + "movabs r9, 15401443493111205", + "mov QWORD PTR [rsi+72], r9", + "movabs r9, 31185040284548497", + "mov QWORD PTR [rsi+80], r9", + "movabs r9, 26937467947448680", + "mov QWORD PTR [rsi+88], r9", + "movabs r9, 19416172761943511", + "mov QWORD PTR [rsi+96], r9", + "movabs r9, 21916122901808376", + "mov QWORD PTR [rsi+104], r9", + "movabs r9, 35910200088776757", + "mov QWORD PTR [rsi+112], r9", + "movabs r9, 1202612321726977", + "mov QWORD PTR [rsi+120], r9", + "movabs r9, 411354790447719", + "mov QWORD PTR [rsi+128], r9", + "movabs r9, 15163111458665677", + "mov QWORD PTR [rsi+136], r9", + "movabs r9, 20565458766169348", + "mov QWORD PTR [rsi+144], r9", + "movabs r9, 16816682460325845", + "mov QWORD PTR [rsi+152], r9", + "movabs r9, 22920956268049798", + "mov QWORD PTR [rsi+160], r9", + "movabs r9, 23677166863944342", + "mov QWORD PTR [rsi+168], r9", + "movabs r9, 34703121006855168", + "mov QWORD PTR [rsi+176], r9", + "movabs r9, 33677345376425628", + "mov QWORD PTR [rsi+184], r9", + "movabs r9, 28933472397947454", + "mov QWORD PTR [rsi+192], r9", + "movabs r9, 19579390106369566", + "mov QWORD PTR [rsi+200], r9", + "movabs r9, 26799599498134591", + "mov QWORD PTR [rsi+208], r9", + "movabs r9, 15889643835192413", + "mov QWORD PTR [rsi+216], r9", + "movabs r9, 2282148053410728", + "mov QWORD PTR [rsi+224], r9", + "movabs r9, 16668952760323958", + "mov QWORD PTR [rsi+232], r9", + "movabs r9, 25011900965946676", + "mov QWORD PTR [rsi+240], r9", + "movabs r9, 23977247637478740", + "mov QWORD PTR [rsi+248], r9", + "movabs r9, 29427887555995366", + "mov QWORD PTR [rsi+256], r9", + "movabs r9, 22931526182748624", + "mov QWORD PTR [rsi+264], r9", + "movabs r9, 14929091579459166", + "mov QWORD PTR [rsi+272], r9", + "movabs r9, 29185144592086471", + "mov QWORD PTR [rsi+280], r9", + "movabs r9, 8329290213797750", + "mov QWORD PTR [rsi+288], r9", + "movabs r9, 31697782066745459", + "mov QWORD PTR [rsi+296], r9", + "movabs r9, 22432987854353025", + "mov QWORD PTR [rsi+304], r9", + "movabs r9, 545125843890401", + "mov QWORD PTR [rsi+312], r9", + "movabs r9, 31769864501989618", + "mov QWORD PTR [rsi+320], r9", + "movabs r9, 11662103226140216", + "mov QWORD PTR [rsi+328], r9", + "movabs r9, 20130185304250276", + "mov QWORD PTR [rsi+336], r9", + "movabs r9, 25354781094156910", + "mov QWORD PTR [rsi+344], r9", + "movabs r9, 30717142252233347", + "mov QWORD PTR [rsi+352], r9", + "movabs r9, 30375073877558844", + "mov QWORD PTR [rsi+360], r9", + "movabs r9, 5794237307816926", + "mov QWORD PTR [rsi+368], r9", + "movabs r9, 29849966874477923", + "mov QWORD PTR [rsi+376], r9", + "movabs r9, 1137925820308458", + "mov QWORD PTR [rsi+384], r9", + "movabs r9, 13305774323778583", + "mov QWORD PTR [rsi+392], r9", + "movabs r9, 31268732009491712", + "mov QWORD PTR [rsi+400], r9", + "movabs r9, 17002134848261444", + "mov QWORD PTR [rsi+408], r9", + "movabs r9, 35956774717033419", + "mov QWORD PTR [rsi+416], r9", + "movabs r9, 22036141462600008", + "mov QWORD PTR [rsi+424], r9", + "movabs r9, 35087477629023596", + "mov QWORD PTR [rsi+432], r9", + "movabs r9, 30337763489061025", + "mov QWORD PTR [rsi+440], r9", + "movabs r9, 20732429908239468", + "mov QWORD PTR [rsi+448], r9", + "movabs r9, 28041905903253186", + "mov QWORD PTR [rsi+456], r9", + "movabs r9, 35231517950811284", + "mov QWORD PTR [rsi+464], r9", + "movabs r9, 5760968484459269", + "mov QWORD PTR [rsi+472], r9", + "movabs r9, 29186403016613413", + "mov QWORD PTR [rsi+480], r9", + "movabs r9, 29809972144732485", + "mov QWORD PTR [rsi+488], r9", + "movabs r9, 19324591173389987", + "mov QWORD PTR [rsi+496], r9", + "movabs r9, 16492506917666904", + "mov QWORD PTR [rsi+504], r9", + "movabs r9, 14635985826474643", + "mov QWORD PTR [rsi+512], r9", + "movabs r9, 16398082059229396", + "mov QWORD PTR [rsi+520], r9", + "movabs r9, 9638297459285875", + "mov QWORD PTR [rsi+528], r9", + "movabs r9, 20692959164533664", + "mov QWORD PTR [rsi+536], r9", + "movabs r9, 10455835889373941", + "mov QWORD PTR [rsi+544], r9", + "movabs r9, 15088997507073265", + "mov QWORD PTR [rsi+552], r9", + "movabs r9, 19847271512942753", + "mov QWORD PTR [rsi+560], r9", + "movabs r9, 22278162875259735", + "mov QWORD PTR [rsi+568], r9", + "movabs r9, 7984765110959710", + "mov QWORD PTR [rsi+576], r9", + "movabs r9, 3517724245221606", + "mov QWORD PTR [rsi+584], r9", + "movabs r9, 29065087369580419", + "mov QWORD PTR [rsi+592], r9", + "movabs r9, 33749496538019589", + "mov QWORD PTR [rsi+600], r9", + "movabs r9, 22582830675910690", + "mov QWORD PTR [rsi+608], r9", + "movabs r9, 13774157688799364", + "mov QWORD PTR [rsi+616], r9", + "movabs r9, 33738338209470846", + "mov QWORD PTR [rsi+624], r9", + "movabs r9, 20549610337740179", + "mov QWORD PTR [rsi+632], r9", + "movabs r9, 1232604074686745", + "mov QWORD PTR [rsi+640], r9", + "movabs r9, 17645078572608834", + "mov QWORD PTR [rsi+648], r9", + "movabs r9, 21638646536106727", + "mov QWORD PTR [rsi+656], r9", + "movabs r9, 872067341384903", + "mov QWORD PTR [rsi+664], r9", + "movabs r9, 11559822875647717", + "mov QWORD PTR [rsi+672], r9", + "movabs r9, 5433172289935931", + "mov QWORD PTR [rsi+680], r9", + "movabs r9, 5358487101890844", + "mov QWORD PTR [rsi+688], r9", + "movabs r9, 6854656137752657", + "mov QWORD PTR [rsi+696], r9", + "movabs r9, 5370830838457625", + "mov QWORD PTR [rsi+704], r9", + "movabs r9, 20753904747165841", + "mov QWORD PTR [rsi+712], r9", + "movabs r9, 8027804982718602", + "mov QWORD PTR [rsi+720], r9", + "movabs r9, 31479735164668747", + "mov QWORD PTR [rsi+728], r9", + "movabs r9, 5314055668205759", + "mov QWORD PTR [rsi+736], r9", + "movabs r9, 29850847345983039", + "mov QWORD PTR [rsi+744], r9", + "movabs r9, 5636951310400997", + "mov QWORD PTR [rsi+752], r9", + "movabs r9, 27564133741392515", + "mov QWORD PTR [rsi+760], r9", + "movabs r9, 8233800205883732", + "mov QWORD PTR [rsi+768], r9", + "movabs r9, 30088883024233849", + "mov QWORD PTR [rsi+776], r9", + "movabs r9, 3338009929245701", + "mov QWORD PTR [rsi+784], r9", + "movabs r9, 14628791756398056", + "mov QWORD PTR [rsi+792], r9", + "movabs r9, 23830870862829877", + "mov QWORD PTR [rsi+800], r9", + "movabs r9, 28061014216302585", + "mov QWORD PTR [rsi+808], r9", + "movabs r9, 19997999096164636", + "mov QWORD PTR [rsi+816], r9", + "movabs r9, 19771555530215640", + "mov QWORD PTR [rsi+824], r9", + "movabs r9, 10447056982320729", + "mov QWORD PTR [rsi+832], r9", + "movabs r9, 35286145636332471", + "mov QWORD PTR [rsi+840], r9", + "movabs r9, 14470225858518424", + "mov QWORD PTR [rsi+848], r9", + "movabs r9, 30807937853347003", + "mov QWORD PTR [rsi+856], r9", + "movabs r9, 699409659546815", + "mov QWORD PTR [rsi+864], r9", + "movabs r9, 12945035726727688", + "mov QWORD PTR [rsi+872], r9", + "movabs r9, 7098008983067813", + "mov QWORD PTR [rsi+880], r9", + "movabs r9, 28266511219523944", + "mov QWORD PTR [rsi+888], r9", + "movabs r9, 15135022374814013", + "mov QWORD PTR [rsi+896], r9", + "movabs r9, 1158610381635861", + "mov QWORD PTR [rsi+904], r9", + "movabs r9, 31802227079365879", + "mov QWORD PTR [rsi+912], r9", + "movabs r9, 2027559572878823", + "mov QWORD PTR [rsi+920], r9", + "movabs r9, 7402805639339334", + "mov QWORD PTR [rsi+928], r9", + "movabs r9, 8205002449640623", + "mov QWORD PTR [rsi+936], r9", + "movabs r9, 31250542829661849", + "mov QWORD PTR [rsi+944], r9", + "movabs r9, 19527171898598875", + "mov QWORD PTR [rsi+952], r9", + "movabs r9, 26390134497937253", + "mov QWORD PTR [rsi+960], r9", + "movabs r9, 26173917256840158", + "mov QWORD PTR [rsi+968], r9", + "movabs r9, 31797902045269139", + "mov QWORD PTR [rsi+976], r9", + "movabs r9, 20765007236602922", + "mov QWORD PTR [rsi+984], r9", + "movabs r9, 16834811519265361", + "mov QWORD PTR [rsi+992], r9", + "movabs r9, 30142973844857679", + "mov QWORD PTR [rsi+1000], r9", + "movabs r9, 6014775284471242", + "mov QWORD PTR [rsi+1008], r9", + "movabs r9, 8490214048855735", + "mov QWORD PTR [rsi+1016], r9", + "mov eax, 8380417", + "movq xmm15, rax", + "pshufd xmm15, xmm15, 0", + "mov eax, -58728449", + "movq xmm14, rax", + "pshufd xmm14, xmm14, 0", + "mov rdx, rdi", + "mov r8, rsi", "add r8, 4", - "mov edi, 1", + "mov eax, 1", "20:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, 4", - "mov ecx, 128", + "mov ecx, 32", "21:", - "mov eax, DWORD PTR [rsi+512]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov eax, DWORD PTR [rsi]", - "mov edx, eax", - "add edx, 8380417", - "sub edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi+512], edx", - "add eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mov DWORD PTR [rsi], eax", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+512]", + "pshufd xmm4, xmm1, 245", + "pmuludq xmm1, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm1", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm1, xmm2", + "psrlq xmm1, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm1, xmm4", + "psubd xmm1, xmm15", + "movdqa xmm2, xmm1", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm1, xmm2", + "movdqa xmm3, xmm0", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "psubd xmm3, xmm1", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+512], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 21b", - "add rsi, 512", - "sub rdi, 1", + "add rdx, 512", + "sub rax, 1", "jne 20b", - "sub rsi, 1024", - "mov edi, 2", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 8", + "mov eax, 2", "22:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, 4", - "mov ecx, 64", + "mov ecx, 16", "23:", - "mov eax, DWORD PTR [rsi+256]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov eax, DWORD PTR [rsi]", - "mov edx, eax", - "add edx, 8380417", - "sub edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi+256], edx", - "add eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mov DWORD PTR [rsi], eax", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+256]", + "pshufd xmm4, xmm1, 245", + "pmuludq xmm1, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm1", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm1, xmm2", + "psrlq xmm1, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm1, xmm4", + "psubd xmm1, xmm15", + "movdqa xmm2, xmm1", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm1, xmm2", + "movdqa xmm3, xmm0", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "psubd xmm3, xmm1", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+256], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 23b", - "add rsi, 256", - "sub rdi, 1", + "add rdx, 256", + "sub rax, 1", "jne 22b", - "sub rsi, 1024", - "mov edi, 4", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 16", + "mov eax, 4", "24:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, 4", - "mov ecx, 32", + "mov ecx, 8", "25:", - "mov eax, DWORD PTR [rsi+128]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov eax, DWORD PTR [rsi]", - "mov edx, eax", - "add edx, 8380417", - "sub edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi+128], edx", - "add eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mov DWORD PTR [rsi], eax", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+128]", + "pshufd xmm4, xmm1, 245", + "pmuludq xmm1, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm1", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm1, xmm2", + "psrlq xmm1, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm1, xmm4", + "psubd xmm1, xmm15", + "movdqa xmm2, xmm1", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm1, xmm2", + "movdqa xmm3, xmm0", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "psubd xmm3, xmm1", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+128], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 25b", - "add rsi, 128", - "sub rdi, 1", + "add rdx, 128", + "sub rax, 1", "jne 24b", - "sub rsi, 1024", - "mov edi, 8", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 32", + "mov eax, 8", "26:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, 4", - "mov ecx, 16", + "mov ecx, 4", "27:", - "mov eax, DWORD PTR [rsi+64]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov eax, DWORD PTR [rsi]", - "mov edx, eax", - "add edx, 8380417", - "sub edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi+64], edx", - "add eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mov DWORD PTR [rsi], eax", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+64]", + "pshufd xmm4, xmm1, 245", + "pmuludq xmm1, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm1", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm1, xmm2", + "psrlq xmm1, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm1, xmm4", + "psubd xmm1, xmm15", + "movdqa xmm2, xmm1", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm1, xmm2", + "movdqa xmm3, xmm0", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "psubd xmm3, xmm1", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+64], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 27b", - "add rsi, 64", - "sub rdi, 1", + "add rdx, 64", + "sub rax, 1", "jne 26b", - "sub rsi, 1024", - "mov edi, 16", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 64", + "mov eax, 16", "28:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, 4", - "mov ecx, 8", + "mov ecx, 2", "29:", - "mov eax, DWORD PTR [rsi+32]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov eax, DWORD PTR [rsi]", - "mov edx, eax", - "add edx, 8380417", - "sub edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi+32], edx", - "add eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mov DWORD PTR [rsi], eax", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+32]", + "pshufd xmm4, xmm1, 245", + "pmuludq xmm1, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm1", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm1, xmm2", + "psrlq xmm1, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm1, xmm4", + "psubd xmm1, xmm15", + "movdqa xmm2, xmm1", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm1, xmm2", + "movdqa xmm3, xmm0", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "psubd xmm3, xmm1", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+32], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 29b", - "add rsi, 32", - "sub rdi, 1", + "add rdx, 32", + "sub rax, 1", "jne 28b", - "sub rsi, 1024", - "mov edi, 32", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 128", + "mov eax, 32", "210:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, 4", - "mov ecx, 4", + "mov ecx, 1", "211:", - "mov eax, DWORD PTR [rsi+16]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov eax, DWORD PTR [rsi]", - "mov edx, eax", - "add edx, 8380417", - "sub edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi+16], edx", - "add eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mov DWORD PTR [rsi], eax", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+16]", + "pshufd xmm4, xmm1, 245", + "pmuludq xmm1, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm1", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm1, xmm2", + "psrlq xmm1, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm1, xmm4", + "psubd xmm1, xmm15", + "movdqa xmm2, xmm1", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm1, xmm2", + "movdqa xmm3, xmm0", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "psubd xmm3, xmm1", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+16], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 211b", - "add rsi, 16", - "sub rdi, 1", + "add rdx, 16", + "sub rax, 1", "jne 210b", - "sub rsi, 1024", - "mov edi, 64", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 256", + "mov ecx, 32", "212:", - "mov r9d, DWORD PTR [r8]", - "add r8, 4", - "mov ecx, 2", - "213:", - "mov eax, DWORD PTR [rsi+8]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov eax, DWORD PTR [rsi]", - "mov edx, eax", - "add edx, 8380417", - "sub edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi+8], edx", - "add eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mov DWORD PTR [rsi], eax", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+16]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 80", + "pshufd xmm12, xmm13, 245", + "add r8, 8", + "movdqa xmm2, xmm0", + "punpcklqdq xmm0, xmm1", + "punpckhqdq xmm2, xmm1", + "movdqa xmm1, xmm2", + "pshufd xmm4, xmm1, 245", + "pmuludq xmm1, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm1", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm1, xmm2", + "psrlq xmm1, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm1, xmm4", + "psubd xmm1, xmm15", + "movdqa xmm2, xmm1", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm1, xmm2", + "movdqa xmm3, xmm0", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "psubd xmm3, xmm1", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqa xmm1, xmm0", + "punpcklqdq xmm0, xmm3", + "punpckhqdq xmm1, xmm3", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+16], xmm1", + "add rdx, 32", "sub rcx, 1", - "jne 213b", - "add rsi, 8", - "sub rdi, 1", "jne 212b", - "sub rsi, 1024", - "mov edi, 128", - "214:", - "mov r9d, DWORD PTR [r8]", - "add r8, 4", - "mov ecx, 1", - "215:", - "mov eax, DWORD PTR [rsi+4]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov eax, DWORD PTR [rsi]", - "mov edx, eax", - "add edx, 8380417", - "sub edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi+4], edx", - "add eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mov DWORD PTR [rsi], eax", - "add rsi, 4", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 512", + "mov ecx, 32", + "213:", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm2, XMMWORD PTR [rdx+16]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 228", + "pshufd xmm12, xmm13, 245", + "add r8, 16", + "pshufd xmm0, xmm0, 216", + "pshufd xmm2, xmm2, 216", + "movdqa xmm1, xmm0", + "punpcklqdq xmm0, xmm2", + "punpckhqdq xmm1, xmm2", + "pshufd xmm4, xmm1, 245", + "pmuludq xmm1, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm1", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm1, xmm2", + "psrlq xmm1, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm1, xmm4", + "psubd xmm1, xmm15", + "movdqa xmm2, xmm1", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm1, xmm2", + "movdqa xmm3, xmm0", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "psubd xmm3, xmm1", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqa xmm1, xmm0", + "punpckldq xmm0, xmm3", + "punpckhdq xmm1, xmm3", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+16], xmm1", + "add rdx, 32", "sub rcx, 1", - "jne 215b", - "add rsi, 4", - "sub rdi, 1", - "jne 214b", - "sub rsi, 1024", + "jne 213b", + "lfence", + "mov DWORD PTR [rsi+768], r11d", + "ldmxcsr DWORD PTR [rsi+768]", "ret", ) } @@ -871,7 +712,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_ntt(f: *mut [u32; 256], scratch: * /// /// Contract: `VG.Spec.MlDsa.nttInvContract`. Constant time: only the pointers may affect timing, not the data. /// -/// The function stores a table of the 256 negated zetas in `scratch`. +/// The function computes on four coefficients at a time in SSE2 registers, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. /// /// # Safety /// @@ -884,870 +725,697 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_ntt(f: *mut [u32; 256], scratch: * #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa_inv_ntt(f: *mut [u32; 256], scratch: *mut [u64; 128]) { core::arch::naked_asm!( - "mov r9, rsi", - "mov eax, 8380416", - "mov DWORD PTR [r9], eax", - "mov eax, 3572223", - "mov DWORD PTR [r9+4], eax", - "mov eax, 4614810", - "mov DWORD PTR [r9+8], eax", - "mov eax, 4618904", - "mov DWORD PTR [r9+12], eax", - "mov eax, 3201494", - "mov DWORD PTR [r9+16], eax", - "mov eax, 2883726", - "mov DWORD PTR [r9+20], eax", - "mov eax, 3145678", - "mov DWORD PTR [r9+24], eax", - "mov eax, 3201430", - "mov DWORD PTR [r9+28], eax", - "mov eax, 601683", - "mov DWORD PTR [r9+32], eax", - "mov eax, 4837932", - "mov DWORD PTR [r9+36], eax", - "mov eax, 5698129", - "mov DWORD PTR [r9+40], eax", - "mov eax, 6250525", - "mov DWORD PTR [r9+44], eax", - "mov eax, 4615550", - "mov DWORD PTR [r9+48], eax", - "mov eax, 1005239", - "mov DWORD PTR [r9+52], eax", - "mov eax, 7822959", - "mov DWORD PTR [r9+56], eax", - "mov eax, 1221177", - "mov DWORD PTR [r9+60], eax", - "mov eax, 3370349", - "mov DWORD PTR [r9+64], eax", - "mov eax, 4063053", - "mov DWORD PTR [r9+68], eax", - "mov eax, 5717039", - "mov DWORD PTR [r9+72], eax", - "mov eax, 1674615", - "mov DWORD PTR [r9+76], eax", - "mov eax, 3524442", - "mov DWORD PTR [r9+80], eax", - "mov eax, 434125", - "mov DWORD PTR [r9+84], eax", - "mov eax, 7703827", - "mov DWORD PTR [r9+88], eax", - "mov eax, 1335936", - "mov DWORD PTR [r9+92], eax", - "mov eax, 3227876", - "mov DWORD PTR [r9+96], eax", - "mov eax, 6666122", - "mov DWORD PTR [r9+100], eax", - "mov eax, 5926434", - "mov DWORD PTR [r9+104], eax", - "mov eax, 6919699", - "mov DWORD PTR [r9+108], eax", - "mov eax, 642628", - "mov DWORD PTR [r9+112], eax", - "mov eax, 3585098", - "mov DWORD PTR [r9+116], eax", - "mov eax, 5564778", - "mov DWORD PTR [r9+120], eax", - "mov eax, 6096684", - "mov DWORD PTR [r9+124], eax", - "mov eax, 4778199", - "mov DWORD PTR [r9+128], eax", - "mov eax, 5197539", - "mov DWORD PTR [r9+132], eax", - "mov eax, 5639874", - "mov DWORD PTR [r9+136], eax", - "mov eax, 3586446", - "mov DWORD PTR [r9+140], eax", - "mov eax, 3110818", - "mov DWORD PTR [r9+144], eax", - "mov eax, 6279007", - "mov DWORD PTR [r9+148], eax", - "mov eax, 4675594", - "mov DWORD PTR [r9+152], eax", - "mov eax, 7220542", - "mov DWORD PTR [r9+156], eax", - "mov eax, 7986269", - "mov DWORD PTR [r9+160], eax", - "mov eax, 7451668", - "mov DWORD PTR [r9+164], eax", - "mov eax, 7284949", - "mov DWORD PTR [r9+168], eax", - "mov eax, 3506380", - "mov DWORD PTR [r9+172], eax", - "mov eax, 6308588", - "mov DWORD PTR [r9+176], eax", - "mov eax, 4018989", - "mov DWORD PTR [r9+180], eax", - "mov eax, 5138445", - "mov DWORD PTR [r9+184], eax", - "mov eax, 6224367", - "mov DWORD PTR [r9+188], eax", - "mov eax, 4965348", - "mov DWORD PTR [r9+192], eax", - "mov eax, 6621070", - "mov DWORD PTR [r9+196], eax", - "mov eax, 817536", - "mov DWORD PTR [r9+200], eax", - "mov eax, 3574466", - "mov DWORD PTR [r9+204], eax", - "mov eax, 4623627", - "mov DWORD PTR [r9+208], eax", - "mov eax, 1935799", - "mov DWORD PTR [r9+212], eax", - "mov eax, 1716988", - "mov DWORD PTR [r9+216], eax", - "mov eax, 3950053", - "mov DWORD PTR [r9+220], eax", - "mov eax, 2897314", - "mov DWORD PTR [r9+224], eax", - "mov eax, 5188063", - "mov DWORD PTR [r9+228], eax", - "mov eax, 7823561", - "mov DWORD PTR [r9+232], eax", - "mov eax, 4510100", - "mov DWORD PTR [r9+236], eax", - "mov eax, 5463079", - "mov DWORD PTR [r9+240], eax", - "mov eax, 6526611", - "mov DWORD PTR [r9+244], eax", - "mov eax, 5034454", - "mov DWORD PTR [r9+248], eax", - "mov eax, 6522001", - "mov DWORD PTR [r9+252], eax", - "mov eax, 5307408", - "mov DWORD PTR [r9+256], eax", - "mov eax, 7102792", - "mov DWORD PTR [r9+260], eax", - "mov eax, 2635473", - "mov DWORD PTR [r9+264], eax", - "mov eax, 4528402", - "mov DWORD PTR [r9+268], eax", - "mov eax, 4197045", - "mov DWORD PTR [r9+272], eax", - "mov eax, 3222807", - "mov DWORD PTR [r9+276], eax", - "mov eax, 3121440", - "mov DWORD PTR [r9+280], eax", - "mov eax, 274060", - "mov DWORD PTR [r9+284], eax", - "mov eax, 5871437", - "mov DWORD PTR [r9+288], eax", - "mov eax, 6352299", - "mov DWORD PTR [r9+292], eax", - "mov eax, 6442847", - "mov DWORD PTR [r9+296], eax", - "mov eax, 3815725", - "mov DWORD PTR [r9+300], eax", - "mov eax, 5569126", - "mov DWORD PTR [r9+304], eax", - "mov eax, 2983781", - "mov DWORD PTR [r9+308], eax", - "mov eax, 1109516", - "mov DWORD PTR [r9+312], eax", - "mov eax, 4222329", - "mov DWORD PTR [r9+316], eax", - "mov eax, 6852351", - "mov DWORD PTR [r9+320], eax", - "mov eax, 7897768", - "mov DWORD PTR [r9+324], eax", - "mov eax, 7231559", - "mov DWORD PTR [r9+328], eax", - "mov eax, 2962264", - "mov DWORD PTR [r9+332], eax", - "mov eax, 565603", - "mov DWORD PTR [r9+336], eax", - "mov eax, 8210729", - "mov DWORD PTR [r9+340], eax", - "mov eax, 5917973", - "mov DWORD PTR [r9+344], eax", - "mov eax, 3334383", - "mov DWORD PTR [r9+348], eax", - "mov eax, 4166425", - "mov DWORD PTR [r9+352], eax", - "mov eax, 3488383", - "mov DWORD PTR [r9+356], eax", - "mov eax, 6392603", - "mov DWORD PTR [r9+360], eax", - "mov eax, 3197248", - "mov DWORD PTR [r9+364], eax", - "mov eax, 6644104", - "mov DWORD PTR [r9+368], eax", - "mov eax, 8145010", - "mov DWORD PTR [r9+372], eax", - "mov eax, 3250154", - "mov DWORD PTR [r9+376], eax", - "mov eax, 5121960", - "mov DWORD PTR [r9+380], eax", - "mov eax, 2579253", - "mov DWORD PTR [r9+384], eax", - "mov eax, 6592474", - "mov DWORD PTR [r9+388], eax", - "mov eax, 2391089", - "mov DWORD PTR [r9+392], eax", - "mov eax, 2254727", - "mov DWORD PTR [r9+396], eax", - "mov eax, 4898211", - "mov DWORD PTR [r9+400], eax", - "mov eax, 4182915", - "mov DWORD PTR [r9+404], eax", - "mov eax, 1300016", - "mov DWORD PTR [r9+408], eax", - "mov eax, 2362063", - "mov DWORD PTR [r9+412], eax", - "mov eax, 1317678", - "mov DWORD PTR [r9+416], eax", - "mov eax, 5919030", - "mov DWORD PTR [r9+420], eax", - "mov eax, 5344437", - "mov DWORD PTR [r9+424], eax", - "mov eax, 7759253", - "mov DWORD PTR [r9+428], eax", - "mov eax, 4478945", - "mov DWORD PTR [r9+432], eax", - "mov eax, 1226661", - "mov DWORD PTR [r9+436], eax", - "mov eax, 5454601", - "mov DWORD PTR [r9+440], eax", - "mov eax, 5006167", - "mov DWORD PTR [r9+444], eax", - "mov eax, 7023969", - "mov DWORD PTR [r9+448], eax", - "mov eax, 2775755", - "mov DWORD PTR [r9+452], eax", - "mov eax, 5697147", - "mov DWORD PTR [r9+456], eax", - "mov eax, 2778788", - "mov DWORD PTR [r9+460], eax", - "mov eax, 3467665", - "mov DWORD PTR [r9+464], eax", - "mov eax, 6067579", - "mov DWORD PTR [r9+468], eax", - "mov eax, 653275", - "mov DWORD PTR [r9+472], eax", - "mov eax, 459163", - "mov DWORD PTR [r9+476], eax", - "mov eax, 8031605", - "mov DWORD PTR [r9+480], eax", - "mov eax, 327848", - "mov DWORD PTR [r9+484], eax", - "mov eax, 7369194", - "mov DWORD PTR [r9+488], eax", - "mov eax, 2354215", - "mov DWORD PTR [r9+492], eax", - "mov eax, 3818627", - "mov DWORD PTR [r9+496], eax", - "mov eax, 1922253", - "mov DWORD PTR [r9+500], eax", - "mov eax, 2236726", - "mov DWORD PTR [r9+504], eax", - "mov eax, 6635910", - "mov DWORD PTR [r9+508], eax", - "mov eax, 8378664", - "mov DWORD PTR [r9+512], eax", - "mov eax, 1935420", - "mov DWORD PTR [r9+516], eax", - "mov eax, 2659525", - "mov DWORD PTR [r9+520], eax", - "mov eax, 1455890", - "mov DWORD PTR [r9+524], eax", - "mov eax, 5720009", - "mov DWORD PTR [r9+528], eax", - "mov eax, 1780227", - "mov DWORD PTR [r9+532], eax", - "mov eax, 59148", - "mov DWORD PTR [r9+536], eax", - "mov eax, 5607817", - "mov DWORD PTR [r9+540], eax", - "mov eax, 7198174", - "mov DWORD PTR [r9+544], eax", - "mov eax, 8293209", - "mov DWORD PTR [r9+548], eax", - "mov eax, 7743490", - "mov DWORD PTR [r9+552], eax", - "mov eax, 3965306", - "mov DWORD PTR [r9+556], eax", - "mov eax, 3956745", - "mov DWORD PTR [r9+560], eax", - "mov eax, 2296397", - "mov DWORD PTR [r9+564], eax", - "mov eax, 3284915", - "mov DWORD PTR [r9+568], eax", - "mov eax, 3716946", - "mov DWORD PTR [r9+572], eax", - "mov eax, 27812", - "mov DWORD PTR [r9+576], eax", - "mov eax, 7557876", - "mov DWORD PTR [r9+580], eax", - "mov eax, 7371052", - "mov DWORD PTR [r9+584], eax", - "mov eax, 2454145", - "mov DWORD PTR [r9+588], eax", - "mov eax, 1979497", - "mov DWORD PTR [r9+592], eax", - "mov eax, 6783595", - "mov DWORD PTR [r9+596], eax", - "mov eax, 3956944", - "mov DWORD PTR [r9+600], eax", - "mov eax, 3759465", - "mov DWORD PTR [r9+604], eax", - "mov eax, 1685153", - "mov DWORD PTR [r9+608], eax", - "mov eax, 3410568", - "mov DWORD PTR [r9+612], eax", - "mov eax, 5702139", - "mov DWORD PTR [r9+616], eax", - "mov eax, 3768948", - "mov DWORD PTR [r9+620], eax", - "mov eax, 3551006", - "mov DWORD PTR [r9+624], eax", - "mov eax, 7744461", - "mov DWORD PTR [r9+628], eax", - "mov eax, 250446", - "mov DWORD PTR [r9+632], eax", - "mov eax, 2455377", - "mov DWORD PTR [r9+636], eax", - "mov eax, 4146264", - "mov DWORD PTR [r9+640], eax", - "mov eax, 1772588", - "mov DWORD PTR [r9+644], eax", - "mov eax, 6187479", - "mov DWORD PTR [r9+648], eax", - "mov eax, 1727088", - "mov DWORD PTR [r9+652], eax", - "mov eax, 5992904", - "mov DWORD PTR [r9+656], eax", - "mov eax, 3611750", - "mov DWORD PTR [r9+660], eax", - "mov eax, 268456", - "mov DWORD PTR [r9+664], eax", - "mov eax, 3180456", - "mov DWORD PTR [r9+668], eax", - "mov eax, 4633167", - "mov DWORD PTR [r9+672], eax", - "mov eax, 6084318", - "mov DWORD PTR [r9+676], eax", - "mov eax, 7140506", - "mov DWORD PTR [r9+680], eax", - "mov eax, 3838479", - "mov DWORD PTR [r9+684], eax", - "mov eax, 5184741", - "mov DWORD PTR [r9+688], eax", - "mov eax, 5737437", - "mov DWORD PTR [r9+692], eax", - "mov eax, 7126227", - "mov DWORD PTR [r9+696], eax", - "mov eax, 12417", - "mov DWORD PTR [r9+700], eax", - "mov eax, 5382198", - "mov DWORD PTR [r9+704], eax", - "mov eax, 8238582", - "mov DWORD PTR [r9+708], eax", - "mov eax, 89301", - "mov DWORD PTR [r9+712], eax", - "mov eax, 5867399", - "mov DWORD PTR [r9+716], eax", - "mov eax, 1354892", - "mov DWORD PTR [r9+720], eax", - "mov eax, 7767179", - "mov DWORD PTR [r9+724], eax", - "mov eax, 1310261", - "mov DWORD PTR [r9+728], eax", - "mov eax, 2218467", - "mov DWORD PTR [r9+732], eax", - "mov eax, 458740", - "mov DWORD PTR [r9+736], eax", - "mov eax, 1921994", - "mov DWORD PTR [r9+740], eax", - "mov eax, 4340221", - "mov DWORD PTR [r9+744], eax", - "mov eax, 3472069", - "mov DWORD PTR [r9+748], eax", - "mov eax, 6341273", - "mov DWORD PTR [r9+752], eax", - "mov eax, 1879878", - "mov DWORD PTR [r9+756], eax", - "mov eax, 818761", - "mov DWORD PTR [r9+760], eax", - "mov eax, 2178965", - "mov DWORD PTR [r9+764], eax", - "mov eax, 1623354", - "mov DWORD PTR [r9+768], eax", - "mov eax, 6275131", - "mov DWORD PTR [r9+772], eax", - "mov eax, 2374402", - "mov DWORD PTR [r9+776], eax", - "mov eax, 2033807", - "mov DWORD PTR [r9+780], eax", - "mov eax, 7794176", - "mov DWORD PTR [r9+784], eax", - "mov eax, 1179613", - "mov DWORD PTR [r9+788], eax", - "mov eax, 7852436", - "mov DWORD PTR [r9+792], eax", - "mov eax, 2743411", - "mov DWORD PTR [r9+796], eax", - "mov eax, 1476985", - "mov DWORD PTR [r9+800], eax", - "mov eax, 6386371", - "mov DWORD PTR [r9+804], eax", - "mov eax, 5889092", - "mov DWORD PTR [r9+808], eax", - "mov eax, 1393159", - "mov DWORD PTR [r9+812], eax", - "mov eax, 7872490", - "mov DWORD PTR [r9+816], eax", - "mov eax, 1187885", - "mov DWORD PTR [r9+820], eax", - "mov eax, 724804", - "mov DWORD PTR [r9+824], eax", - "mov eax, 1834526", - "mov DWORD PTR [r9+828], eax", - "mov eax, 3033742", - "mov DWORD PTR [r9+832], eax", - "mov eax, 338420", - "mov DWORD PTR [r9+836], eax", - "mov eax, 5732423", - "mov DWORD PTR [r9+840], eax", - "mov eax, 5370669", - "mov DWORD PTR [r9+844], eax", - "mov eax, 2612853", - "mov DWORD PTR [r9+848], eax", - "mov eax, 4231948", - "mov DWORD PTR [r9+852], eax", - "mov eax, 7630840", - "mov DWORD PTR [r9+856], eax", - "mov eax, 4022750", - "mov DWORD PTR [r9+860], eax", - "mov eax, 4399818", - "mov DWORD PTR [r9+864], eax", - "mov eax, 5811406", - "mov DWORD PTR [r9+868], eax", - "mov eax, 1615530", - "mov DWORD PTR [r9+872], eax", - "mov eax, 6657188", - "mov DWORD PTR [r9+876], eax", - "mov eax, 6715099", - "mov DWORD PTR [r9+880], eax", - "mov eax, 6352379", - "mov DWORD PTR [r9+884], eax", - "mov eax, 7216819", - "mov DWORD PTR [r9+888], eax", - "mov eax, 3369273", - "mov DWORD PTR [r9+892], eax", - "mov eax, 4385746", - "mov DWORD PTR [r9+896], eax", - "mov eax, 11879", - "mov DWORD PTR [r9+900], eax", - "mov eax, 1370517", - "mov DWORD PTR [r9+904], eax", - "mov eax, 5360024", - "mov DWORD PTR [r9+908], eax", - "mov eax, 5016875", - "mov DWORD PTR [r9+912], eax", - "mov eax, 8165537", - "mov DWORD PTR [r9+916], eax", - "mov eax, 7835041", - "mov DWORD PTR [r9+920], eax", - "mov eax, 770441", - "mov DWORD PTR [r9+924], eax", - "mov eax, 5274859", - "mov DWORD PTR [r9+928], eax", - "mov eax, 1103344", - "mov DWORD PTR [r9+932], eax", - "mov eax, 7872272", - "mov DWORD PTR [r9+936], eax", - "mov eax, 553718", - "mov DWORD PTR [r9+940], eax", - "mov eax, 7520273", - "mov DWORD PTR [r9+944], eax", - "mov eax, 4949981", - "mov DWORD PTR [r9+948], eax", - "mov eax, 8240173", - "mov DWORD PTR [r9+952], eax", - "mov eax, 1514152", - "mov DWORD PTR [r9+956], eax", - "mov eax, 2185084", - "mov DWORD PTR [r9+960], eax", - "mov eax, 5256655", - "mov DWORD PTR [r9+964], eax", - "mov eax, 6022044", - "mov DWORD PTR [r9+968], eax", - "mov eax, 2193087", - "mov DWORD PTR [r9+972], eax", - "mov eax, 3014420", - "mov DWORD PTR [r9+976], eax", - "mov eax, 1716814", - "mov DWORD PTR [r9+980], eax", - "mov eax, 5454363", - "mov DWORD PTR [r9+984], eax", - "mov eax, 392707", - "mov DWORD PTR [r9+988], eax", - "mov eax, 303005", - "mov DWORD PTR [r9+992], eax", - "mov eax, 4849188", - "mov DWORD PTR [r9+996], eax", - "mov eax, 3974485", - "mov DWORD PTR [r9+1000], eax", - "mov eax, 3773731", - "mov DWORD PTR [r9+1004], eax", - "mov eax, 6480365", - "mov DWORD PTR [r9+1008], eax", - "mov eax, 781875", - "mov DWORD PTR [r9+1012], eax", - "mov eax, 7325939", - "mov DWORD PTR [r9+1016], eax", - "mov eax, 731434", - "mov DWORD PTR [r9+1020], eax", - "mov rsi, rdi", - "mov r8, r9", - "add r8, 1020", - "mov edi, 128", + "stmxcsr DWORD PTR [rsi+768]", + "mov r11d, DWORD PTR [rsi+768]", + "and r11d, 65535", + "mov eax, 8127", + "mov DWORD PTR [rsi+772], eax", + "ldmxcsr DWORD PTR [rsi+772]", + "lfence", + "movabs r9, 111012023893504", + "mov QWORD PTR [rsi], r9", + "movabs r9, 33764919763013891", + "mov QWORD PTR [rsi+8], r9", + "movabs r9, 32652304184483396", + "mov QWORD PTR [rsi+16], r9", + "movabs r9, 2003464812134697", + "mov QWORD PTR [rsi+24], r9", + "movabs r9, 10107995079564843", + "mov QWORD PTR [rsi+32], r9", + "movabs r9, 27008953388524718", + "mov QWORD PTR [rsi+40], r9", + "movabs r9, 23603259066260085", + "mov QWORD PTR [rsi+48], r9", + "movabs r9, 11510954738022985", + "mov QWORD PTR [rsi+56], r9", + "movabs r9, 4398527550166616", + "mov QWORD PTR [rsi+64], r9", + "movabs r9, 15401443493111205", + "mov QWORD PTR [rsi+72], r9", + "movabs r9, 31185040284548497", + "mov QWORD PTR [rsi+80], r9", + "movabs r9, 26937467947448680", + "mov QWORD PTR [rsi+88], r9", + "movabs r9, 19416172761943511", + "mov QWORD PTR [rsi+96], r9", + "movabs r9, 21916122901808376", + "mov QWORD PTR [rsi+104], r9", + "movabs r9, 35910200088776757", + "mov QWORD PTR [rsi+112], r9", + "movabs r9, 1202612321726977", + "mov QWORD PTR [rsi+120], r9", + "movabs r9, 411354790447719", + "mov QWORD PTR [rsi+128], r9", + "movabs r9, 15163111458665677", + "mov QWORD PTR [rsi+136], r9", + "movabs r9, 20565458766169348", + "mov QWORD PTR [rsi+144], r9", + "movabs r9, 16816682460325845", + "mov QWORD PTR [rsi+152], r9", + "movabs r9, 22920956268049798", + "mov QWORD PTR [rsi+160], r9", + "movabs r9, 23677166863944342", + "mov QWORD PTR [rsi+168], r9", + "movabs r9, 34703121006855168", + "mov QWORD PTR [rsi+176], r9", + "movabs r9, 33677345376425628", + "mov QWORD PTR [rsi+184], r9", + "movabs r9, 28933472397947454", + "mov QWORD PTR [rsi+192], r9", + "movabs r9, 19579390106369566", + "mov QWORD PTR [rsi+200], r9", + "movabs r9, 26799599498134591", + "mov QWORD PTR [rsi+208], r9", + "movabs r9, 15889643835192413", + "mov QWORD PTR [rsi+216], r9", + "movabs r9, 2282148053410728", + "mov QWORD PTR [rsi+224], r9", + "movabs r9, 16668952760323958", + "mov QWORD PTR [rsi+232], r9", + "movabs r9, 25011900965946676", + "mov QWORD PTR [rsi+240], r9", + "movabs r9, 23977247637478740", + "mov QWORD PTR [rsi+248], r9", + "movabs r9, 29427887555995366", + "mov QWORD PTR [rsi+256], r9", + "movabs r9, 22931526182748624", + "mov QWORD PTR [rsi+264], r9", + "movabs r9, 14929091579459166", + "mov QWORD PTR [rsi+272], r9", + "movabs r9, 29185144592086471", + "mov QWORD PTR [rsi+280], r9", + "movabs r9, 8329290213797750", + "mov QWORD PTR [rsi+288], r9", + "movabs r9, 31697782066745459", + "mov QWORD PTR [rsi+296], r9", + "movabs r9, 22432987854353025", + "mov QWORD PTR [rsi+304], r9", + "movabs r9, 545125843890401", + "mov QWORD PTR [rsi+312], r9", + "movabs r9, 31769864501989618", + "mov QWORD PTR [rsi+320], r9", + "movabs r9, 11662103226140216", + "mov QWORD PTR [rsi+328], r9", + "movabs r9, 20130185304250276", + "mov QWORD PTR [rsi+336], r9", + "movabs r9, 25354781094156910", + "mov QWORD PTR [rsi+344], r9", + "movabs r9, 30717142252233347", + "mov QWORD PTR [rsi+352], r9", + "movabs r9, 30375073877558844", + "mov QWORD PTR [rsi+360], r9", + "movabs r9, 5794237307816926", + "mov QWORD PTR [rsi+368], r9", + "movabs r9, 29849966874477923", + "mov QWORD PTR [rsi+376], r9", + "movabs r9, 1137925820308458", + "mov QWORD PTR [rsi+384], r9", + "movabs r9, 13305774323778583", + "mov QWORD PTR [rsi+392], r9", + "movabs r9, 31268732009491712", + "mov QWORD PTR [rsi+400], r9", + "movabs r9, 17002134848261444", + "mov QWORD PTR [rsi+408], r9", + "movabs r9, 35956774717033419", + "mov QWORD PTR [rsi+416], r9", + "movabs r9, 22036141462600008", + "mov QWORD PTR [rsi+424], r9", + "movabs r9, 35087477629023596", + "mov QWORD PTR [rsi+432], r9", + "movabs r9, 30337763489061025", + "mov QWORD PTR [rsi+440], r9", + "movabs r9, 20732429908239468", + "mov QWORD PTR [rsi+448], r9", + "movabs r9, 28041905903253186", + "mov QWORD PTR [rsi+456], r9", + "movabs r9, 35231517950811284", + "mov QWORD PTR [rsi+464], r9", + "movabs r9, 5760968484459269", + "mov QWORD PTR [rsi+472], r9", + "movabs r9, 29186403016613413", + "mov QWORD PTR [rsi+480], r9", + "movabs r9, 29809972144732485", + "mov QWORD PTR [rsi+488], r9", + "movabs r9, 19324591173389987", + "mov QWORD PTR [rsi+496], r9", + "movabs r9, 16492506917666904", + "mov QWORD PTR [rsi+504], r9", + "movabs r9, 14635985826474643", + "mov QWORD PTR [rsi+512], r9", + "movabs r9, 16398082059229396", + "mov QWORD PTR [rsi+520], r9", + "movabs r9, 9638297459285875", + "mov QWORD PTR [rsi+528], r9", + "movabs r9, 20692959164533664", + "mov QWORD PTR [rsi+536], r9", + "movabs r9, 10455835889373941", + "mov QWORD PTR [rsi+544], r9", + "movabs r9, 15088997507073265", + "mov QWORD PTR [rsi+552], r9", + "movabs r9, 19847271512942753", + "mov QWORD PTR [rsi+560], r9", + "movabs r9, 22278162875259735", + "mov QWORD PTR [rsi+568], r9", + "movabs r9, 7984765110959710", + "mov QWORD PTR [rsi+576], r9", + "movabs r9, 3517724245221606", + "mov QWORD PTR [rsi+584], r9", + "movabs r9, 29065087369580419", + "mov QWORD PTR [rsi+592], r9", + "movabs r9, 33749496538019589", + "mov QWORD PTR [rsi+600], r9", + "movabs r9, 22582830675910690", + "mov QWORD PTR [rsi+608], r9", + "movabs r9, 13774157688799364", + "mov QWORD PTR [rsi+616], r9", + "movabs r9, 33738338209470846", + "mov QWORD PTR [rsi+624], r9", + "movabs r9, 20549610337740179", + "mov QWORD PTR [rsi+632], r9", + "movabs r9, 1232604074686745", + "mov QWORD PTR [rsi+640], r9", + "movabs r9, 17645078572608834", + "mov QWORD PTR [rsi+648], r9", + "movabs r9, 21638646536106727", + "mov QWORD PTR [rsi+656], r9", + "movabs r9, 872067341384903", + "mov QWORD PTR [rsi+664], r9", + "movabs r9, 11559822875647717", + "mov QWORD PTR [rsi+672], r9", + "movabs r9, 5433172289935931", + "mov QWORD PTR [rsi+680], r9", + "movabs r9, 5358487101890844", + "mov QWORD PTR [rsi+688], r9", + "movabs r9, 6854656137752657", + "mov QWORD PTR [rsi+696], r9", + "movabs r9, 5370830838457625", + "mov QWORD PTR [rsi+704], r9", + "movabs r9, 20753904747165841", + "mov QWORD PTR [rsi+712], r9", + "movabs r9, 8027804982718602", + "mov QWORD PTR [rsi+720], r9", + "movabs r9, 31479735164668747", + "mov QWORD PTR [rsi+728], r9", + "movabs r9, 5314055668205759", + "mov QWORD PTR [rsi+736], r9", + "movabs r9, 29850847345983039", + "mov QWORD PTR [rsi+744], r9", + "movabs r9, 5636951310400997", + "mov QWORD PTR [rsi+752], r9", + "movabs r9, 27564133741392515", + "mov QWORD PTR [rsi+760], r9", + "movabs r9, 8233800205883732", + "mov QWORD PTR [rsi+768], r9", + "movabs r9, 30088883024233849", + "mov QWORD PTR [rsi+776], r9", + "movabs r9, 3338009929245701", + "mov QWORD PTR [rsi+784], r9", + "movabs r9, 14628791756398056", + "mov QWORD PTR [rsi+792], r9", + "movabs r9, 23830870862829877", + "mov QWORD PTR [rsi+800], r9", + "movabs r9, 28061014216302585", + "mov QWORD PTR [rsi+808], r9", + "movabs r9, 19997999096164636", + "mov QWORD PTR [rsi+816], r9", + "movabs r9, 19771555530215640", + "mov QWORD PTR [rsi+824], r9", + "movabs r9, 10447056982320729", + "mov QWORD PTR [rsi+832], r9", + "movabs r9, 35286145636332471", + "mov QWORD PTR [rsi+840], r9", + "movabs r9, 14470225858518424", + "mov QWORD PTR [rsi+848], r9", + "movabs r9, 30807937853347003", + "mov QWORD PTR [rsi+856], r9", + "movabs r9, 699409659546815", + "mov QWORD PTR [rsi+864], r9", + "movabs r9, 12945035726727688", + "mov QWORD PTR [rsi+872], r9", + "movabs r9, 7098008983067813", + "mov QWORD PTR [rsi+880], r9", + "movabs r9, 28266511219523944", + "mov QWORD PTR [rsi+888], r9", + "movabs r9, 15135022374814013", + "mov QWORD PTR [rsi+896], r9", + "movabs r9, 1158610381635861", + "mov QWORD PTR [rsi+904], r9", + "movabs r9, 31802227079365879", + "mov QWORD PTR [rsi+912], r9", + "movabs r9, 2027559572878823", + "mov QWORD PTR [rsi+920], r9", + "movabs r9, 7402805639339334", + "mov QWORD PTR [rsi+928], r9", + "movabs r9, 8205002449640623", + "mov QWORD PTR [rsi+936], r9", + "movabs r9, 31250542829661849", + "mov QWORD PTR [rsi+944], r9", + "movabs r9, 19527171898598875", + "mov QWORD PTR [rsi+952], r9", + "movabs r9, 26390134497937253", + "mov QWORD PTR [rsi+960], r9", + "movabs r9, 26173917256840158", + "mov QWORD PTR [rsi+968], r9", + "movabs r9, 31797902045269139", + "mov QWORD PTR [rsi+976], r9", + "movabs r9, 20765007236602922", + "mov QWORD PTR [rsi+984], r9", + "movabs r9, 16834811519265361", + "mov QWORD PTR [rsi+992], r9", + "movabs r9, 30142973844857679", + "mov QWORD PTR [rsi+1000], r9", + "movabs r9, 6014775284471242", + "mov QWORD PTR [rsi+1008], r9", + "movabs r9, 8490214048855735", + "mov QWORD PTR [rsi+1016], r9", + "mov eax, 8380417", + "movq xmm15, rax", + "pshufd xmm15, xmm15, 0", + "mov eax, -58728449", + "movq xmm14, rax", + "pshufd xmm14, xmm14, 0", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 1008", + "mov ecx, 32", "20:", - "mov r9d, DWORD PTR [r8]", - "add r8, -4", - "mov ecx, 1", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm2, XMMWORD PTR [rdx+16]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 27", + "pshufd xmm12, xmm13, 245", + "add r8, -16", + "pshufd xmm0, xmm0, 216", + "pshufd xmm2, xmm2, 216", + "movdqa xmm1, xmm0", + "punpcklqdq xmm0, xmm2", + "punpckhqdq xmm1, xmm2", + "movdqa xmm3, xmm1", + "psubd xmm3, xmm0", + "paddd xmm3, xmm15", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqa xmm1, xmm0", + "punpckldq xmm0, xmm3", + "punpckhdq xmm1, xmm3", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+16], xmm1", + "add rdx, 32", + "sub rcx, 1", + "jne 20b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 504", + "mov ecx, 32", "21:", - "mov eax, DWORD PTR [rsi]", - "mov r10d, DWORD PTR [rsi+4]", - "mov edx, eax", - "add edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi], edx", - "add eax, 8380417", - "sub eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rsi+4], r10d", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+16]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 5", + "pshufd xmm12, xmm13, 245", + "add r8, -8", + "movdqa xmm2, xmm0", + "punpcklqdq xmm0, xmm1", + "punpckhqdq xmm2, xmm1", + "movdqa xmm1, xmm2", + "movdqa xmm3, xmm1", + "psubd xmm3, xmm0", + "paddd xmm3, xmm15", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqa xmm1, xmm0", + "punpcklqdq xmm0, xmm3", + "punpckhqdq xmm1, xmm3", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+16], xmm1", + "add rdx, 32", "sub rcx, 1", "jne 21b", - "add rsi, 4", - "sub rdi, 1", - "jne 20b", - "sub rsi, 1024", - "mov edi, 64", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 252", + "mov eax, 32", "22:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, -4", - "mov ecx, 2", + "mov ecx, 1", "23:", - "mov eax, DWORD PTR [rsi]", - "mov r10d, DWORD PTR [rsi+8]", - "mov edx, eax", - "add edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi], edx", - "add eax, 8380417", - "sub eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rsi+8], r10d", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+16]", + "movdqa xmm3, xmm1", + "psubd xmm3, xmm0", + "paddd xmm3, xmm15", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+16], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 23b", - "add rsi, 8", - "sub rdi, 1", + "add rdx, 16", + "sub rax, 1", "jne 22b", - "sub rsi, 1024", - "mov edi, 32", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 124", + "mov eax, 16", "24:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, -4", - "mov ecx, 4", + "mov ecx, 2", "25:", - "mov eax, DWORD PTR [rsi]", - "mov r10d, DWORD PTR [rsi+16]", - "mov edx, eax", - "add edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi], edx", - "add eax, 8380417", - "sub eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rsi+16], r10d", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+32]", + "movdqa xmm3, xmm1", + "psubd xmm3, xmm0", + "paddd xmm3, xmm15", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+32], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 25b", - "add rsi, 16", - "sub rdi, 1", + "add rdx, 32", + "sub rax, 1", "jne 24b", - "sub rsi, 1024", - "mov edi, 16", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 60", + "mov eax, 8", "26:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, -4", - "mov ecx, 8", + "mov ecx, 4", "27:", - "mov eax, DWORD PTR [rsi]", - "mov r10d, DWORD PTR [rsi+32]", - "mov edx, eax", - "add edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi], edx", - "add eax, 8380417", - "sub eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rsi+32], r10d", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+64]", + "movdqa xmm3, xmm1", + "psubd xmm3, xmm0", + "paddd xmm3, xmm15", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+64], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 27b", - "add rsi, 32", - "sub rdi, 1", + "add rdx, 64", + "sub rax, 1", "jne 26b", - "sub rsi, 1024", - "mov edi, 8", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 28", + "mov eax, 4", "28:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, -4", - "mov ecx, 16", + "mov ecx, 8", "29:", - "mov eax, DWORD PTR [rsi]", - "mov r10d, DWORD PTR [rsi+64]", - "mov edx, eax", - "add edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi], edx", - "add eax, 8380417", - "sub eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rsi+64], r10d", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+128]", + "movdqa xmm3, xmm1", + "psubd xmm3, xmm0", + "paddd xmm3, xmm15", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+128], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 29b", - "add rsi, 64", - "sub rdi, 1", + "add rdx, 128", + "sub rax, 1", "jne 28b", - "sub rsi, 1024", - "mov edi, 4", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 12", + "mov eax, 2", "210:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, -4", - "mov ecx, 32", + "mov ecx, 16", "211:", - "mov eax, DWORD PTR [rsi]", - "mov r10d, DWORD PTR [rsi+128]", - "mov edx, eax", - "add edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi], edx", - "add eax, 8380417", - "sub eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rsi+128], r10d", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+256]", + "movdqa xmm3, xmm1", + "psubd xmm3, xmm0", + "paddd xmm3, xmm15", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+256], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 211b", - "add rsi, 128", - "sub rdi, 1", + "add rdx, 256", + "sub rax, 1", "jne 210b", - "sub rsi, 1024", - "mov edi, 2", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 4", + "mov eax, 1", "212:", - "mov r9d, DWORD PTR [r8]", + "movdqu xmm13, XMMWORD PTR [r8]", + "pshufd xmm13, xmm13, 0", + "pshufd xmm12, xmm13, 245", "add r8, -4", - "mov ecx, 64", + "mov ecx, 32", "213:", - "mov eax, DWORD PTR [rsi]", - "mov r10d, DWORD PTR [rsi+256]", - "mov edx, eax", - "add edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi], edx", - "add eax, 8380417", - "sub eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rsi+256], r10d", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdx]", + "movdqu xmm1, XMMWORD PTR [rdx+512]", + "movdqa xmm3, xmm1", + "psubd xmm3, xmm0", + "paddd xmm3, xmm15", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm0", + "movdqu XMMWORD PTR [rdx+512], xmm3", + "add rdx, 16", "sub rcx, 1", "jne 213b", - "add rsi, 256", - "sub rdi, 1", + "add rdx, 512", + "sub rax, 1", "jne 212b", - "sub rsi, 1024", - "mov edi, 1", + "mov rdx, rdi", + "mov eax, 16382", + "movq xmm13, rax", + "pshufd xmm13, xmm13, 0", + "movdqa xmm12, xmm13", + "mov ecx, 64", "214:", - "mov r9d, DWORD PTR [r8]", - "add r8, -4", - "mov ecx, 128", - "215:", - "mov eax, DWORD PTR [rsi]", - "mov r10d, DWORD PTR [rsi+512]", - "mov edx, eax", - "add edx, r10d", - "sub edx, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add edx, r11d", - "mov DWORD PTR [rsi], edx", - "add eax, 8380417", - "sub eax, r10d", - "sub eax, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add eax, r11d", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rsi+512], r10d", - "add rsi, 4", + "movdqu xmm3, XMMWORD PTR [rdx]", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdx], xmm3", + "add rdx, 16", "sub rcx, 1", - "jne 215b", - "add rsi, 512", - "sub rdi, 1", "jne 214b", - "sub rsi, 1024", - "mov r9d, 8347681", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rsi]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rsi], r10d", - "add rsi, 4", - "sub rcx, 1", - "jne 216b", + "lfence", + "mov DWORD PTR [rsi+768], r11d", + "ldmxcsr DWORD PTR [rsi+768]", "ret", ) } From ae7eb79564f854750ddeecb53ab8f7e0bf4d17ed Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 14:36:01 +0000 Subject: [PATCH 2/8] ML-DSA on x86-64: polynomial arithmetic in SSE2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vg_mldsa_multiply_ntt, vg_mldsa_multiply_add_ntt, vg_mldsa_add and vg_mldsa_sub now compute on four coefficients at a time in SSE2 registers, with the vector helpers of the NTT (Vec.lean). The products are two Montgomery multiplications: f·g·2⁻³², then by 2⁶⁴ mod q, which is f·g in [0, 2q), reduced with vcsub (for multiply_add, h is then added and the sum reduced). They use pmuludq, so they run inside ML-KEM's withMxcsr. These functions have no working space and use no stack, so MXCSR goes through the last 8 bytes of h, addressed through r8 = h. The last four coefficients of h are loaded into xmm6 first. The loop stores the first 252 coefficients. The last four are computed from registers before MXCSR is loaded back, and stored after it. Their callers' proofs are unchanged: the functions still need no stack and never write rsp. add and sub use paddd/psubd and vcsub/vcadd. Proofs: the lanes of a product (mul_lane, mulAdd_lane), the loop (Mul.step, Mul.loop_ok), the last block (Mul.last), the function (Mul.fn_ok), and withMxcsr through the end of a polynomial (withMxcsrH_ok). AddSub is reproven the same way. ML-DSA-65 instructions executed (callgrind, PR #464 -> this): sign -19%, verify -11%, keygen -2%. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- README.md | 6 +- docs/algorithms/ml-dsa-44.toml | 2 +- docs/algorithms/ml-dsa-65.toml | 2 +- docs/algorithms/ml-dsa-87.toml | 2 +- .../Artifacts/MlDsaArith/X86_64.lean | 10 + .../Impl/MlDsa/X86_64/Arith/AddSub.lean | 36 +- .../Impl/MlDsa/X86_64/Arith/Mul.lean | 66 ++- .../Proof/MlDsa/X86_64/Arith/AddSub.lean | 307 +++++----- .../Proof/MlDsa/X86_64/Arith/Mul.lean | 548 +++++++++++------- .../Proof/MlDsa/X86_64/Arith/Mxcsr.lean | 73 +++ src/asm/x86_64/mldsa.rs | 306 ++++++++-- 11 files changed, 918 insertions(+), 440 deletions(-) create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean diff --git a/README.md b/README.md index 00030ddd4..a14742701 100644 --- a/README.md +++ b/README.md @@ -774,7 +774,7 @@ yours to keep: ✅ -✅ SSE2 NTT +✅ SSE2 polynomial arithmetic ✅ SHA extensions @@ -790,7 +790,7 @@ yours to keep: ✅ -✅ SSE2 NTT +✅ SSE2 polynomial arithmetic ✅ SHA extensions @@ -806,7 +806,7 @@ yours to keep: ✅ -✅ SSE2 NTT +✅ SSE2 polynomial arithmetic ✅ SHA extensions diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml index 7f316830d..854e6670e 100644 --- a/docs/algorithms/ml-dsa-44.toml +++ b/docs/algorithms/ml-dsa-44.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa44.rs"] asm = ["mldsa44", "mldsa"] -optimized = { x86_64 = "SSE2 NTT" } +optimized = { x86_64 = "SSE2 polynomial arithmetic" } diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml index 26de285e2..e87725611 100644 --- a/docs/algorithms/ml-dsa-65.toml +++ b/docs/algorithms/ml-dsa-65.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa65.rs"] asm = ["mldsa65", "mldsa"] -optimized = { x86_64 = "SSE2 NTT" } +optimized = { x86_64 = "SSE2 polynomial arithmetic" } diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml index 1f76a641b..9cfbc9f3a 100644 --- a/docs/algorithms/ml-dsa-87.toml +++ b/docs/algorithms/ml-dsa-87.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa87.rs"] asm = ["mldsa87", "mldsa"] -optimized = { x86_64 = "SSE2 NTT" } +optimized = { x86_64 = "SSE2 polynomial arithmetic" } diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean index f8fdd427a..a3d95256b 100644 --- a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean +++ b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean @@ -46,6 +46,10 @@ def artifacts : List Artifact := [ { Spec.MlDsa.mulApi with target := X86_64.target doc := Spec.MlDsa.mulApi.doc + (notes := ["The function computes on four coefficients at a time in SSE2 registers. It sets MXCSR to \ + `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), \ + through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before \ + returning."]) code := Impl.MlDsa.X86_64.Arith.mul contract := Spec.MlDsa.mulContract X86_64.abi verified := Proof.MlDsa.X86_64.Arith.mul_verified @@ -53,6 +57,10 @@ def artifacts : List Artifact := [ { Spec.MlDsa.mulAddApi with target := X86_64.target doc := Spec.MlDsa.mulAddApi.doc + (notes := ["The function computes on four coefficients at a time in SSE2 registers. It sets MXCSR to \ + `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), \ + through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before \ + returning."]) code := Impl.MlDsa.X86_64.Arith.mulAdd contract := Spec.MlDsa.mulAddContract X86_64.abi verified := Proof.MlDsa.X86_64.Arith.mulAdd_verified @@ -60,6 +68,7 @@ def artifacts : List Artifact := [ { Spec.MlDsa.addApi with target := X86_64.target doc := Spec.MlDsa.addApi.doc + (notes := ["The function computes on four coefficients at a time in SSE2 registers."]) code := Impl.MlDsa.X86_64.Arith.add contract := Spec.MlDsa.addContract X86_64.abi verified := Proof.MlDsa.X86_64.Arith.add_verified @@ -67,6 +76,7 @@ def artifacts : List Artifact := [ { Spec.MlDsa.subApi with target := X86_64.target doc := Spec.MlDsa.subApi.doc + (notes := ["The function computes on four coefficients at a time in SSE2 registers."]) code := Impl.MlDsa.X86_64.Arith.sub contract := Spec.MlDsa.subContract X86_64.abi verified := Proof.MlDsa.X86_64.Arith.sub_verified diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/AddSub.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/AddSub.lean index 1ea537f64..3383b263c 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/AddSub.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/AddSub.lean @@ -1,33 +1,39 @@ -import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Common +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Vec /-! # ML-DSA on x86-64: `vg_mldsa_add` and `vg_mldsa_sub` -`add(f = rdi, g = rsi)` and `sub(f = rdi, g = rsi)` run over the 256 -coefficients with `rdi` and `rsi` pointing at coefficient `i` of `f` and -`g`, and `rcx = 256 - i` counting down: `f[i] + g[i]` (for `sub`, -`f[i] + q - g[i]`), less than `2q`, is reduced with `csubQ` and stored to -`f[i]`. Every address and branch depends only on the pointers. +`add(f = rdi, g = rsi)` and `sub(f = rdi, g = rsi)` compute on four +coefficients at a time, as doublewords of SSE registers (see `Vec.lean`), +with `rdi` and `rsi` pointing at coefficient `4i` of `f` and `g`, and +`rcx = 64 - i` counting down: `f + g`, less than `2q`, is reduced with +`vcsub` (for `sub`, `f - g`, in `(-q, q)`, with `vcadd`) and stored to `f`, +with `q` in the doublewords of `xmm15`. Every address and branch depends +only on the pointers. -/ namespace VG.Impl.MlDsa.X86_64.Arith open VG.X86_64 +open VG.Impl.MlKem.X86_64 (xb rcxLoop) -/-- Advance the two pointers and count down. -/ -def step2 : List Instr := - [.alu .add .rdi (.imm 4), .alu .add .rsi (.imm 4), .alu .sub .rcx (.imm 1)] +/-- `q` in the doublewords of `xmm15`, through `rax`. -/ +def qPro : List Instr := [.mov32 .rax (.imm 8380417), .xop (.movq .xmm15 .rax), .xop (.pshufd .xmm15 .xmm15 0)] + +/-- Store `xmm0` to `f` and advance the two pointers. -/ +def accTail : List Instr := + [.movdquStore (at_ .rdi 0) .xmm0, .alu .add .rdi (.imm 16), .alu .add .rsi (.imm 16)] def addBody : List Instr := - [.mov32 .rax (.mem (at_ .rdi 0)), .alu32 .add .rax (.mem (at_ .rsi 0))] ++ csubQ .rax .rdx ++ - [.store32 (at_ .rdi 0) .rax] ++ step2 + [.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0), xb .paddd .xmm0 .xmm1] ++ + vcsub .xmm0 .xmm2 ++ accTail def subBody : List Instr := - [.mov32 .rax (.mem (at_ .rdi 0)), .alu32 .add .rax (.imm qImm), .alu32 .sub .rax (.mem (at_ .rsi 0))] ++ - csubQ .rax .rdx ++ [.store32 (at_ .rdi 0) .rax] ++ step2 + [.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0), xb .psubd .xmm0 .xmm1] ++ + vcadd .xmm0 .xmm2 ++ accTail -def add : Prog isa := .seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block addBody) .ne) +def add : Prog isa := .seq (.block qPro) (rcxLoop 64 addBody) -def sub : Prog isa := .seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block subBody) .ne) +def sub : Prog isa := .seq (.block qPro) (rcxLoop 64 subBody) end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Mul.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Mul.lean index bd0679661..59f8d7b71 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Mul.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Mul.lean @@ -1,41 +1,63 @@ -import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Common +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Vec /-! # ML-DSA on x86-64: `vg_mldsa_multiply_ntt` and `vg_mldsa_multiply_add_ntt` `multiplyNTT(h = rdi, f = rsi, g = rdx)` and `multiplyAddNTT(h = rdi, -f = rsi, g = rdx)` run over the 256 coefficients with `rdi`, `rsi` and `r8` -(`g`, as `mul` writes `rdx`) pointing at coefficient `i` of `h`, `f` and -`g`, and `rcx = 256 - i` counting down: the product `f[i] · g[i]` (by -`mul`, less than `q²`; for `multiplyAddNTT`, plus `h[i]`) is reduced with -`reduce` and stored to `h[i]`. Every address and branch depends only on the -pointers. +f = rsi, g = rdx)` compute on four coefficients at a time, as doublewords of +SSE registers (see `Vec.lean`), with `rdi`, `rsi` and `rdx` pointing at +coefficient `4i` of `h`, `f` and `g`, and `rcx = 63 - i` counting down: +`vmont` of `f` by `g` is `f · g · 2⁻³²`, and `vmont` of that by +`2⁶⁴ mod q = 2365951` (in `xmm11`) is `f · g`, less than `2q`, which `vcsub` +reduces (for `multiplyAddNTT`, `h` is then added and the sum reduced), and +it is stored to `h`. + +The functions have no working space and use no stack, so `withMxcsr` saves +MXCSR through the last 8 bytes of `h` (`r8 = h`): the last four coefficients +of `h` are loaded to `xmm6` first, the loop stores the first 252, the last +four are computed from registers before MXCSR is loaded back, and stored +after it. Every address and branch depends only on the pointers. -/ namespace VG.Impl.MlDsa.X86_64.Arith open VG.X86_64 +open VG.Impl.MlKem.X86_64 (xb xmov withMxcsr rcxLoop) -/-- Advance the three pointers and count down. -/ -def step3 : List Instr := - [.alu .add .rdi (.imm 4), .alu .add .rsi (.imm 4), .alu .add .r8 (.imm 4), .alu .sub .rcx (.imm 1)] +/-- The constants and `2⁶⁴ mod q` in the doublewords of `xmm11`. -/ +def mulPro : List Instr := + vconsts ++ [.mov32 .rax (.imm 2365951), .xop (.movq .xmm11 .rax), .xop (.pshufd .xmm11 .xmm11 0)] -/-- `f[i] · g[i]`, in `rax`. -/ -def mulHead : List Instr := - [.mov32 .rax (.mem (at_ .rsi 0)), .mov32 .r9 (.mem (at_ .r8 0)), .mul .r9] +/-- The coefficients of `f`, `g` and `h` to `xmm3`, `xmm13` and `xmm5`. -/ +def mulLoads : List Instr := + [.movdquLoad .xmm3 (at_ .rsi 0), .movdquLoad .xmm13 (at_ .rdx 0), .movdquLoad .xmm5 (at_ .rdi 0)] -/-- `f[i] · g[i] + h[i]`, in `rax`. -/ -def mulAddHead : List Instr := - mulHead ++ [.mov32 .r9 (.mem (at_ .rdi 0)), .alu .add .rax (.reg .r9)] +/-- `f · g` for four coefficients, in `[0, q)`, in `xmm3`. -/ +def mulCore : List Instr := + [.xop (.pshufd .xmm12 .xmm13 0xF5)] ++ vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ + vmont .xmm3 .xmm11 .xmm11 .xmm2 .xmm4 ++ vcsub .xmm3 .xmm2 -def mulBody : List Instr := mulHead ++ reduce ++ [.store32 (at_ .rdi 0) .r10] ++ step3 +/-- `h + f · g` for four coefficients, in `[0, q)`, in `xmm3`. -/ +def mulAddCore : List Instr := mulCore ++ [xb .paddd .xmm3 .xmm5] ++ vcsub .xmm3 .xmm2 -def mulAddBody : List Instr := mulAddHead ++ reduce ++ [.store32 (at_ .rdi 0) .r10] ++ step3 +/-- Store `xmm3` to `h` and advance the three pointers. -/ +def mulTail : List Instr := + [.movdquStore (at_ .rdi 0) .xmm3, .alu .add .rdi (.imm 16), .alu .add .rsi (.imm 16), + .alu .add .rdx (.imm 16)] -def mul : Prog isa := - .seq (.block [.mov .r8 (.reg .rdx)]) (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block mulBody) .ne)) +/-- The last four coefficients, with those of `h` in `xmm6`, to `xmm3`. -/ +def mulLast (core : List Instr) : List Instr := + [.movdquLoad .xmm3 (at_ .rsi 0), .movdquLoad .xmm13 (at_ .rdx 0), xmov .xmm5 .xmm6] ++ core -def mulAdd : Prog isa := - .seq (.block [.mov .r8 (.reg .rdx)]) (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block mulAddBody) .ne)) +/-- A function of `h`, `f` and `g` four coefficients at a time by `core`. -/ +def mulFn (core : List Instr) : Prog isa := + .seq (.block [.mov .r8 (.reg .rdi), .movdquLoad .xmm6 (at_ .rdi 1008)]) + (.seq (withMxcsr .r8 1016 + (.seq (.block mulPro) (.seq (rcxLoop 63 (mulLoads ++ core ++ mulTail)) (.block (mulLast core))))) + (.block [.movdquStore (at_ .rdi 0) .xmm3])) + +def mul : Prog isa := mulFn mulCore + +def mulAdd : Prog isa := mulFn mulAddCore end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/AddSub.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/AddSub.lean index 96b1bc978..bf574a3af 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/AddSub.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/AddSub.lean @@ -1,149 +1,194 @@ import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.AddSub +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Basic import VerifiedGarbage.Proof.Framework.X86_64.Abi /-! # ML-DSA on x86-64: `vg_mldsa_add` and `vg_mldsa_sub` -Untrusted: everything here is checked by Lean. +Untrusted: everything here is checked by Lean. Each iteration of the loop +stores four coefficients to `f` (`addBody_ok`, `subBody_ok`), each `csubL` +of the sum (`caddL` of the difference), whose value is `addD_toNat` +(`subD_toNat`); the loop leaves `f` with all 256 (`AddSub.fn_ok`). -/ namespace VG.Proof.MlDsa.X86_64.Arith open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith open VG.Proof.MlDsa.Arith -open VG.Proof.MlKem.X86_64 (Keep WP.keep writesOnly gprPreserved_of wp_countdown ifp ifn ptr_step) +open VG.Proof.MlKem.X86_64 (Keep WP.keep writesOnly gprPreserved_of ifp ifn ptr_step GOnly wp_rcxLoop xmm_setXmm + add_ofNat_zero) +open VG.Impl.MlKem.X86_64 (xb xmov) open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs) -/-! ## One coefficient -/ - -theorem addBody_ok (s : State) (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rdi) 4) - (h2 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) (h3 : InRegions s.wr (s.gpr .rdi) 4) : - WP isa (.block addBody) s fun s' => - (s'.mem = s.mem.writeW (s.gpr .rdi) (csubD (s.mem.readW (s.gpr .rdi) 32 + s.mem.readW (s.gpr .rsi) 32)) ∧ - s'.gpr .rdi = s.gpr .rdi + 4 ∧ s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rax, .rdx, .rdi, .rsi, .rcx] s s' := by - refine WP.keep _ ?_ (by decide) - unfold addBody csubQ step2 - xrund [h1, h2, h3, List.cons_append, List.nil_append, csubD] - -theorem subBody_ok (s : State) (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rdi) 4) - (h2 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) (h3 : InRegions s.wr (s.gpr .rdi) 4) : - WP isa (.block subBody) s fun s' => - (s'.mem = s.mem.writeW (s.gpr .rdi) - (csubD (s.mem.readW (s.gpr .rdi) 32 + qImm - s.mem.readW (s.gpr .rsi) 32)) ∧ - s'.gpr .rdi = s.gpr .rdi + 4 ∧ s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rax, .rdx, .rdi, .rsi, .rcx] s s' := by - refine WP.keep _ ?_ (by decide) - unfold subBody csubQ step2 - xrund [h1, h2, h3, List.cons_append, List.nil_append, csubD] +/-! ## Four coefficients -/ + +/-- What an iteration of `add` stores. -/ +def addV (x y : BitVec 128) : BitVec 128 := csubV (XBinOp.eval .paddd x y) + +/-- What an iteration of `sub` stores. -/ +def subV (x y : BitVec 128) : BitVec 128 := caddV (XBinOp.eval .psubd x y) + +theorem dword_addV (x y : BitVec 128) {i : Nat} (hi : i < 4) : + dword (addV x y) i = csubL (dword x i + dword y i) := by + rw [addV, dword_csubV _ hi, dword_paddd _ _ hi] + +theorem dword_subV (x y : BitVec 128) {i : Nat} (hi : i < 4) : + dword (subV x y) i = caddL (dword x i - dword y i) := by + rw [subV, dword_caddV _ hi, dword_psubd _ _ hi] + +/-- The body of `add` or `sub`: the store of `F x y` of the vectors at `rdi` +and `rsi`, and the counts. -/ +theorem accBody_ok {op : XBinOp} {fix : List Instr} {F : BitVec 128 → BitVec 128 → BitVec 128} + (hF : ∀ (s : State), s.xmm .xmm15 = qV → + WP isa (.block (xb op .xmm0 .xmm1 :: fix)) s fun s' => + s'.xmm .xmm0 = F (s.xmm .xmm0) (s.xmm .xmm1) ∧ s'.gpr = s.gpr ∧ s'.mem = s.mem ∧ s'.rd = s.rd ∧ + s'.wr = s.wr ∧ s'.xmm .xmm15 = qV) + (s : State) (hq : s.xmm .xmm15 = qV) (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rdi) 16) + (h2 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 16) (h3 : InRegions s.wr (s.gpr .rdi) 16) : + WP isa (.block (([.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0)] : List Instr) ++ + ((xb op .xmm0 .xmm1 :: fix) ++ (accTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr))))) s fun s' => + s'.mem = s.mem.writeW (s.gpr .rdi) (F (s.mem.readW (s.gpr .rdi) 128) (s.mem.readW (s.gpr .rsi) 128)) ∧ + s'.gpr .rdi = s.gpr .rdi + 16 ∧ s'.gpr .rsi = s.gpr .rsi + 16 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ + s'.zf = some (s.gpr .rcx - 1 == 0) ∧ s'.rd = s.rd ∧ s'.wr = s.wr ∧ s'.xmm .xmm15 = qV ∧ + Keep [.rdi, .rsi, .rcx] s s' := by + rw [WP.block_append_iff] + vrund [h1, h2] + rw [show Instr.xop (.bin op .xmm0 .xmm1) :: (fix ++ (accTail ++ [.alu .sub .rcx (.imm 1)])) = + (xb op .xmm0 .xmm1 :: fix) ++ (accTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr)) from rfl, + WP.block_append_iff] + refine WP.mono (hF _ (by simp only [xmm_setXmm, reduceCtorEq, ite_false]; exact hq)) + fun s2 ⟨h0, g2, m2, r2, w2, q2⟩ => ?_ + simp only [accTail] + vrund [g2, m2, r2, w2, h3, h0, q2] + refine ⟨fun r hr => ?_, rfl, rfl⟩ + simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr.1, hr.2.1, hr.2.2, ite_false] + +theorem addFix_ok (s : State) (hq : s.xmm .xmm15 = qV) : + WP isa (.block (xb .paddd .xmm0 .xmm1 :: vcsub .xmm0 .xmm2)) s fun s' => + s'.xmm .xmm0 = addV (s.xmm .xmm0) (s.xmm .xmm1) ∧ s'.gpr = s.gpr ∧ s'.mem = s.mem ∧ s'.rd = s.rd ∧ + s'.wr = s.wr ∧ s'.xmm .xmm15 = qV := by + simp only [vcsub, vcadd, xmov, xb] + vrun [eval_movdqa] + rw [hq] + exact ⟨rfl, trivial, trivial, trivial, trivial, rfl⟩ + +theorem subFix_ok (s : State) (hq : s.xmm .xmm15 = qV) : + WP isa (.block (xb .psubd .xmm0 .xmm1 :: vcadd .xmm0 .xmm2)) s fun s' => + s'.xmm .xmm0 = subV (s.xmm .xmm0) (s.xmm .xmm1) ∧ s'.gpr = s.gpr ∧ s'.mem = s.mem ∧ s'.rd = s.rd ∧ + s'.wr = s.wr ∧ s'.xmm .xmm15 = qV := by + simp only [vcadd, xmov, xb] + vrun [eval_movdqa] + rw [hq] + exact ⟨rfl, trivial, trivial, trivial, trivial, rfl⟩ /-! ## The loop -/ namespace AddSub -/-- After `i` coefficients, each one `v k`. -/ +/-- After `i` vectors, each coefficient before `4i` is `v k`. -/ structure Inv (s₀ : State) (v : Nat → BitVec 32) (i : Nat) (s : State) : Prop where - rdi : s.gpr .rdi = s₀.gpr .rdi + BitVec.ofNat 64 (4 * i) - rsi : s.gpr .rsi = s₀.gpr .rsi + BitVec.ofNat 64 (4 * i) + rdi : s.gpr .rdi = s₀.gpr .rdi + BitVec.ofNat 64 (16 * i) + rsi : s.gpr .rsi = s₀.gpr .rsi + BitVec.ofNat 64 (16 * i) rd : s.rd = s₀.rd wr : s.wr = s₀.wr + q : s.xmm .xmm15 = qV frame : Frame [pR (s₀.gpr .rdi)] s₀.mem s.mem - coeff : ∀ k < 256, coeffAt s.mem (s₀.gpr .rdi) k = if k < i then v k else coeffAt s₀.mem (s₀.gpr .rdi) k + coeff : ∀ k < 256, coeffAt s.mem (s₀.gpr .rdi) k = if k < 4 * i then v k else coeffAt s₀.mem (s₀.gpr .rdi) k section variable {t : Poly → Poly → Poly} {s₀ : State} (hp : (accK t).pre s₀) include hp -theorem inF {i : Nat} (hi : i < 256) {s : State} (hrd : s.rd = s₀.rd) (hwr : s.wr = s₀.wr) : - InRegions (s.rd ++ s.wr) (coeffAddr (s₀.gpr .rdi) i) 4 ∧ InRegions s.wr (coeffAddr (s₀.gpr .rdi) i) 4 := by - rw [hrd, hwr, hp.1, hp.2.1] - exact ⟨⟨_, by simp, coeff_contains _ hi⟩, ⟨_, by simp, coeff_contains _ hi⟩⟩ - -theorem inG {i : Nat} (hi : i < 256) {s : State} (hrd : s.rd = s₀.rd) (hwr : s.wr = s₀.wr) : - InRegions (s.rd ++ s.wr) (coeffAddr (s₀.gpr .rsi) i) 4 := by - rw [hrd, hwr, hp.1, hp.2.1] - exact ⟨_, by simp, coeff_contains _ hi⟩ - -/-- `g` is not written. -/ -theorem coeffG {m : Mem} (hf : Frame [pR (s₀.gpr .rdi)] s₀.mem m) {i : Nat} (hi : i < 256) : - coeffAt m (s₀.gpr .rsi) i = coeffAt s₀.mem (s₀.gpr .rsi) i := - coeffAt_frame hf (by simpa using hp.2.2.1.symm) hi - -omit hp in -theorem inv_step {v : Nat → BitVec 32} {i : Nat} (hi : i < 256) {s s' : State} (hI : Inv s₀ v i s) - (hm : s'.mem = s.mem.writeW (s.gpr .rdi) (v i)) (hdi : s'.gpr .rdi = s.gpr .rdi + 4) - (hsi : s'.gpr .rsi = s.gpr .rsi + 4) (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) : - Inv s₀ v (i + 1) s' where - rdi := by rw [hdi, hI.rdi]; exact ptr_step _ i 4 - rsi := by rw [hsi, hI.rsi]; exact ptr_step _ i 4 - rd := hrd.trans hI.rd - wr := hwr.trans hI.wr - frame := by - rw [hm, hI.rdi] - exact hI.frame.writeW (List.mem_singleton_self _) _ (coeff_contains _ hi) - coeff k hk := by - rw [hm, hI.rdi, ← coeffAddr, coeffAt_writeW _ _ hk hi, hI.coeff k hk] - by_cases e : i = k - · subst e; simp - · have : (k < i + 1) = (k < i) := propext (by omega) - simp only [e, this, ↓reduceIte] - -omit hp in -/-- The loop, from the prologue, with a body that stores `v i` to coefficient `i`. -/ -theorem loop_ok {body : List Instr} {v : Nat → BitVec 32} - (hbody : ∀ i < 256, ∀ s, Inv s₀ v i s → WP isa (.block body) s fun s' => - (s'.mem = s.mem.writeW (s.gpr .rdi) (v i) ∧ s'.gpr .rdi = s.gpr .rdi + 4 ∧ - s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rax, .rdx, .rdi, .rsi, .rcx] s s') : - WP isa (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block body) .ne)) s₀ (Inv s₀ v 256) := by - refine WP.seq ?_ - xrund - refine wp_countdown (cnt := .rcx) (N := 256) (by decide) (by decide) (Inv s₀ v) (fun i hi s hI _ => ?_) - (fun _ h => h) ?_ (by simp [VG.Proof.MlKem.X86_64.setReg_gpr]) - · refine WP.mono (hbody i hi s hI) fun s' ⟨⟨hm, hdi, hsi, hcx, hz⟩, hk⟩ => ⟨?_, hcx, hz⟩ - exact inv_step hi hI hm hdi hsi hk.2.1 hk.2.2 - · refine ⟨by simp [VG.Proof.MlKem.X86_64.setReg_gpr], by simp [VG.Proof.MlKem.X86_64.setReg_gpr], rfl, rfl, - Frame.refl _ _, fun k _ => ?_⟩ - simp [VG.Proof.MlKem.X86_64.setReg_mem] - -/-- The coefficients the loop reads. -/ -theorem reads {v : Nat → BitVec 32} {i : Nat} (hi : i < 256) {s : State} (hI : Inv s₀ v i s) : - s.mem.readW (s.gpr .rdi) 32 = coeffAt s₀.mem (s₀.gpr .rdi) i ∧ - s.mem.readW (s.gpr .rsi) 32 = coeffAt s₀.mem (s₀.gpr .rsi) i ∧ - InRegions (s.rd ++ s.wr) (s.gpr .rdi) 4 ∧ InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4 ∧ - InRegions s.wr (s.gpr .rdi) 4 := by - have hf := inF hp hi hI.rd hI.wr - refine ⟨?_, ?_, ?_, ?_, ?_⟩ - · rw [hI.rdi, ← coeffAddr, ← coeffAt_eq, hI.coeff i hi]; simp only [Nat.lt_irrefl, ↓reduceIte] - · rw [hI.rsi, ← coeffAddr, ← coeffAt_eq, coeffG hp hI.frame hi] - · rw [hI.rdi]; exact hf.1 - · rw [hI.rsi]; exact inG hp hi hI.rd hI.wr - · rw [hI.rdi]; exact hf.2 - -omit hp in -/-- The result, with the values of `add` or `sub`. -/ -theorem result {v : Nat → BitVec 32} {r : Poly} {s : State} (hI : Inv s₀ v 256 s) - (hv : ∀ i < 256, (v i).toNat = (r[i]!).val) : PolyIs s.mem (s₀.gpr .rdi) r := - polyIs_of_toNat fun i hi => by rw [hI.coeff i hi]; simp only [hi, ↓reduceIte]; exact hv i hi +/-- An iteration, which stores `F` of the vectors of `f` and `g`, whose +doublewords are `L` of theirs. -/ +theorem step {op : XBinOp} {fix : List Instr} {F : BitVec 128 → BitVec 128 → BitVec 128} + {L : BitVec 32 → BitVec 32 → BitVec 32} + (hF : ∀ (s : State), s.xmm .xmm15 = qV → + WP isa (.block (xb op .xmm0 .xmm1 :: fix)) s fun s' => + s'.xmm .xmm0 = F (s.xmm .xmm0) (s.xmm .xmm1) ∧ s'.gpr = s.gpr ∧ s'.mem = s.mem ∧ s'.rd = s.rd ∧ + s'.wr = s.wr ∧ s'.xmm .xmm15 = qV) + (hL : ∀ x y : BitVec 128, ∀ e < 4, dword (F x y) e = L (dword x e) (dword y e)) + {i : Nat} (hi : i < 64) {s : State} + (hI : Inv s₀ (fun k => L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) i s) : + WP isa (.block (([.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0)] : List Instr) ++ + ((xb op .xmm0 .xmm1 :: fix) ++ (accTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr))))) s fun s' => + Inv s₀ (fun k => L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) (i + 1) s' ∧ + s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by + have j0 : 4 * i + 4 ≤ 256 := by omega + have hw : pR (s₀.gpr .rdi) ∈ s.wr := by rw [hI.wr, hp.2.1]; simp + have hr : pR (s₀.gpr .rsi) ∈ s.rd ++ s.wr := by rw [hI.rd, hI.wr, hp.1]; simp + have e1 : s.gpr .rdi = coeffAddr (s₀.gpr .rdi) (4 * i) := by rw [hI.rdi]; congr 2; omega + have e2 : s.gpr .rsi = coeffAddr (s₀.gpr .rsi) (4 * i) := by rw [hI.rsi]; congr 2; omega + refine WP.mono (accBody_ok hF s hI.q (by rw [e1]; exact f_in (List.mem_append_right _ hw) j0) + (by rw [e2]; exact ⟨_, hr, pR_contains _ j0⟩) (by rw [e1]; exact f_in hw j0)) + fun s' ⟨hm, hdi, hsi, hcx, hz, hrd, hwr, hq, _⟩ => ⟨?_, hcx, hz⟩ + refine ⟨by rw [hdi, hI.rdi]; exact ptr_step _ i 16, by rw [hsi, hI.rsi]; exact ptr_step _ i 16, + hrd.trans hI.rd, hwr.trans hI.wr, hq, ?_, fun k hk => ?_⟩ + · rw [hm, e1]; exact hI.frame.writeW (List.mem_singleton_self _) _ (pR_contains _ j0) + · rw [hm, e1, coeffAt_write128 _ _ j0 _ hk] + split + · rename_i h + rw [ifp (show k < 4 * (i + 1) by omega), hL _ _ _ (by omega), e2, dword_readW _ _ (by omega), + dword_readW _ _ (by omega), coeffAddr_add, coeffAddr_add, ← coeffAt_eq, ← coeffAt_eq, + show 4 * i + (k - 4 * i) = k by omega, hI.coeff k hk, ifn (by omega), + coeffAt_frame hI.frame (by simpa using hp.2.2.1.symm) (by rw [n_eq]; exact hk)] + · rename_i h + rw [hI.coeff k hk] + by_cases h' : k < 4 * i + · rw [ifp h', ifp (by omega)] + · rw [ifn h', ifn (by omega)] /-- The whole function, from its precondition. -/ -theorem fn_ok {body : List Instr} {v : Nat → BitVec 32} - (hbody : ∀ i < 256, ∀ s, Inv s₀ v i s → WP isa (.block body) s fun s' => - (s'.mem = s.mem.writeW (s.gpr .rdi) (v i) ∧ s'.gpr .rdi = s.gpr .rdi + 4 ∧ - s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rax, .rdx, .rdi, .rsi, .rcx] s s') - (hv : ∀ i < 256, (v i).toNat = ((t (polyAt s₀.mem (s₀.gpr .rdi)) (polyAt s₀.mem (s₀.gpr .rsi)))[i]!).val) - (hc : writesOnly [.rax, .rdx, .rdi, .rsi, .rcx] - (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block body) .ne)) = true) - (hm : Code.allInstrs (fun i => !loadsMxcsr i) - (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block body) .ne) : Prog isa) = true) : - ∃ tr s', Exec isa (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block body) .ne)) s₀ tr s' ∧ - abiPreserved s₀ s' ∧ (accK t).post s₀ s' := by - obtain ⟨tr, s', he, hI, hk⟩ := WP.keep _ (loop_ok hbody) hc +theorem fn_ok {op : XBinOp} {fix : List Instr} {F : BitVec 128 → BitVec 128 → BitVec 128} + {L : BitVec 32 → BitVec 32 → BitVec 32} + (hF : ∀ (s : State), s.xmm .xmm15 = qV → + WP isa (.block (xb op .xmm0 .xmm1 :: fix)) s fun s' => + s'.xmm .xmm0 = F (s.xmm .xmm0) (s.xmm .xmm1) ∧ s'.gpr = s.gpr ∧ s'.mem = s.mem ∧ s'.rd = s.rd ∧ + s'.wr = s.wr ∧ s'.xmm .xmm15 = qV) + (hL : ∀ x y : BitVec 128, ∀ e < 4, dword (F x y) e = L (dword x e) (dword y e)) + (hv : ∀ k < 256, (L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)).toNat = + ((t (polyAt s₀.mem (s₀.gpr .rdi)) (polyAt s₀.mem (s₀.gpr .rsi)))[k]!).val) + (hc : writesOnly [.rax, .rdi, .rsi, .rcx] (.seq (.block qPro) (VG.Impl.MlKem.X86_64.rcxLoop 64 + (([.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0)] : List Instr) ++ (xb op .xmm0 .xmm1 :: fix) ++ + accTail))) = true) + (hm : Code.allInstrs (fun i => !loadsMxcsr i) (.seq (.block qPro) (VG.Impl.MlKem.X86_64.rcxLoop 64 + (([.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0)] : List Instr) ++ (xb op .xmm0 .xmm1 :: fix) ++ + accTail)) : Prog isa) = true) : + ∃ tr s', Exec isa (.seq (.block qPro) (VG.Impl.MlKem.X86_64.rcxLoop 64 + (([.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0)] : List Instr) ++ (xb op .xmm0 .xmm1 :: fix) ++ + accTail))) s₀ tr s' ∧ abiPreserved s₀ s' ∧ (accK t).post s₀ s' := by + have hw : pR (s₀.gpr .rdi) ∈ s₀.wr := by rw [hp.2.1]; simp + have hW : WP isa (.seq (.block qPro) (VG.Impl.MlKem.X86_64.rcxLoop 64 + (([.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0)] : List Instr) ++ (xb op .xmm0 .xmm1 :: fix) ++ + accTail))) s₀ + (Inv s₀ (fun k => L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) 64) := by + refine WP.seq (WP.mono (Q := fun (w : State) => w.xmm .xmm15 = qV ∧ Keep [.rax] s₀ w ∧ w.mem = s₀.mem) + (by + simp only [qPro] + vrund + refine ⟨fun r hr => ?_, rfl, rfl⟩ + simp only [List.mem_singleton] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setXmm, hr, ite_false]) fun w ⟨hq, k1, m1⟩ => ?_) + refine wp_rcxLoop (N := 64) (by decide) (by decide) _ (fun u o _ => ⟨?_, ?_, by rw [o.keep.2.1, k1.2.1], + by rw [o.keep.2.2, k1.2.2], by rw [o.xmm]; exact hq, by rw [o.mem, m1]; exact Frame.refl _ _, + fun k _ => by rw [o.mem, m1, ifn (by omega)]⟩) fun i hi u hI => ?_ + · rw [o.keep.gpr (by decide), k1.gpr (by decide), Nat.mul_zero, add_ofNat_zero] + · rw [o.keep.gpr (by decide), k1.gpr (by decide), Nat.mul_zero, add_ofNat_zero] + · rw [show [Instr.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0)] ++ + (xb op .xmm0 .xmm1 :: fix) ++ accTail ++ [.alu .sub .rcx (.imm 1)] = + [.movdquLoad .xmm0 (at_ .rdi 0), .movdquLoad .xmm1 (at_ .rsi 0)] ++ + ((xb op .xmm0 .xmm1 :: fix) ++ (accTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) by + simp only [List.append_assoc]] + exact step hp hF hL hi hI + obtain ⟨tr, s', he, hI, hk⟩ := WP.keep _ hW hc refine ⟨tr, s', he, abiPreserved_of_exec hm he (gprPreserved_of hk (by decide) hI.frame ?_), - result hI hv⟩ - simpa using hp.2.2.2.1 + polyIs_of_toNat fun k hk => ?_⟩ + · simpa using hp.2.2.2.1 + · rw [n_eq] at hk + rw [hI.coeff k hk, ifp (by omega)] + exact hv k hk end @@ -153,27 +198,25 @@ end AddSub theorem add_correct (s : State) (hs : (accK Spec.MlDsa.add).pre s) : ∃ t s', Exec isa Impl.MlDsa.X86_64.Arith.add s t s' ∧ abiPreserved s s' ∧ (accK Spec.MlDsa.add).post s s' := - AddSub.fn_ok hs (v := fun i => csubD (coeffAt s.mem (s.gpr .rdi) i + coeffAt s.mem (s.gpr .rsi) i)) - (fun i hi s' hI => by - obtain ⟨e1, e2, h1, h2, h3⟩ := AddSub.reads hs hi hI - have := addBody_ok s' h1 h2 h3 - rwa [e1, e2] at this) - (fun i hi => by - rw [add_get _ _ hi] - exact csubD_add_val (polyAt_val hs.2.2.2.2.2.1 hi).symm (polyAt_val hs.2.2.2.2.2.2 hi).symm) - (by decide) (by decide) + AddSub.fn_ok hs (op := .paddd) (fix := vcsub .xmm0 .xmm2) (L := fun a b => csubL (a + b)) addFix_ok + (fun x y e he => dword_addV x y he) + (fun k hk => by + have hk' : k < n := by rw [n_eq]; exact hk + rw [add_get _ _ hk', addD_toNat (by rw [← polyAt_val hs.2.2.2.2.2.1 hk']; exact val_lt _) + (by rw [← polyAt_val hs.2.2.2.2.2.2 hk']; exact val_lt _), + ← polyAt_val hs.2.2.2.2.2.1 hk', ← polyAt_val hs.2.2.2.2.2.2 hk', val_add]) + (by decide +kernel) (by decide +kernel) theorem sub_correct (s : State) (hs : (accK Spec.MlDsa.sub).pre s) : ∃ t s', Exec isa Impl.MlDsa.X86_64.Arith.sub s t s' ∧ abiPreserved s s' ∧ (accK Spec.MlDsa.sub).post s s' := - AddSub.fn_ok hs (v := fun i => csubD (coeffAt s.mem (s.gpr .rdi) i + qImm - coeffAt s.mem (s.gpr .rsi) i)) - (fun i hi s' hI => by - obtain ⟨e1, e2, h1, h2, h3⟩ := AddSub.reads hs hi hI - have := subBody_ok s' h1 h2 h3 - rwa [e1, e2] at this) - (fun i hi => by - rw [sub_get _ _ hi] - exact csubD_sub_val (polyAt_val hs.2.2.2.2.2.1 hi).symm (polyAt_val hs.2.2.2.2.2.2 hi).symm) - (by decide) (by decide) + AddSub.fn_ok hs (op := .psubd) (fix := vcadd .xmm0 .xmm2) (L := fun a b => caddL (a - b)) subFix_ok + (fun x y e he => dword_subV x y he) + (fun k hk => by + have hk' : k < n := by rw [n_eq]; exact hk + rw [sub_get _ _ hk', subD_toNat (by rw [← polyAt_val hs.2.2.2.2.2.1 hk']; exact val_lt _) + (by rw [← polyAt_val hs.2.2.2.2.2.2 hk']; exact val_lt _), + ← polyAt_val hs.2.2.2.2.2.1 hk', ← polyAt_val hs.2.2.2.2.2.2 hk', val_sub]) + (by decide +kernel) (by decide +kernel) /-- The pointers and `rsp` are public. -/ def accτ : X86_64.Taint.T := X86_64.Taint.ofRegs [.rdi, .rsi, .rsp] diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean index 3939bb475..7cdf5d3b0 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean @@ -1,205 +1,353 @@ import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Mul -import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Basic -import VerifiedGarbage.Proof.Framework.X86_64.Abi +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mxcsr +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Ntt +import VerifiedGarbage.Proof.Framework.X86_64.Mxcsr /-! # ML-DSA on x86-64: `vg_mldsa_multiply_ntt` and `vg_mldsa_multiply_add_ntt` -Untrusted: everything here is checked by Lean. Both functions are one loop -over the coefficients (`Mul.fn_ok`), whose body stores the reduced product -(`mulBody_ok`), or the reduced sum of the product and the coefficient of -`h` (`mulAddBody_ok`). +Untrusted: everything here is checked by Lean. A doubleword of `mulV x y` +is the product of those of `x` and `y` (`mul_lane`): `mont` of `mont` of +their product by `2⁶⁴ mod q` (`mont_mont_R2`). The loop stores four of them +at a time to the first 252 coefficients of `h` (`Mul.step`, `Mul.loop_ok`), +inside `withMxcsr` through the last 8 bytes of `h`; the last four are +computed from the coefficients of `h` loaded before (`Mul.last`), and stored +after MXCSR is loaded back (`Mul.fn_ok`). -/ namespace VG.Proof.MlDsa.X86_64.Arith open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith open VG.Proof.MlDsa.Arith -open VG.Proof.MlKem.X86_64 (Keep WP.keep writesOnly gprPreserved_of wp_counted ifp ifn ptr_step - toNat_setWidth64) -open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs) - -/-! ## One coefficient -/ - -/-- The product of two words, as `mul` leaves it. -/ -abbrev prod32 (a b : BitVec 32) : BitVec 64 := - BitVec.ofNat 64 ((BitVec.setWidth 64 a).toNat * (BitVec.setWidth 64 b).toNat) - -theorem mulHead_ok (s : State) (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) - (h2 : InRegions (s.rd ++ s.wr) (s.gpr .r8) 4) : - WP isa (.block mulHead) s fun s' => - (s'.gpr .rax = prod32 (s.mem.readW (s.gpr .rsi) 32) (s.mem.readW (s.gpr .r8) 32) ∧ s'.mem = s.mem) ∧ - Keep [.rax, .rdx, .r9] s s' := by - refine WP.keep _ ?_ (by decide) - unfold mulHead - xrund [h1, h2] - -theorem mulAddHead_ok (s : State) (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4) - (h2 : InRegions (s.rd ++ s.wr) (s.gpr .r8) 4) (h3 : InRegions (s.rd ++ s.wr) (s.gpr .rdi) 4) : - WP isa (.block mulAddHead) s fun s' => - (s'.gpr .rax = prod32 (s.mem.readW (s.gpr .rsi) 32) (s.mem.readW (s.gpr .r8) 32) + - BitVec.setWidth 64 (s.mem.readW (s.gpr .rdi) 32) ∧ s'.mem = s.mem) ∧ Keep [.rax, .rdx, .r9] s s' := by - refine WP.keep _ ?_ (by decide) - unfold mulAddHead mulHead - xrund [h1, h2, h3, List.cons_append, List.nil_append] - -theorem mulTail_ok (s : State) (hw : InRegions s.wr (s.gpr .rdi) 4) : - WP isa (.block (([.store32 (at_ .rdi 0) .r10] : List Instr) ++ step3)) s fun s' => - (s'.mem = s.mem.writeW (s.gpr .rdi) (BitVec.setWidth 32 (s.gpr .r10)) ∧ s'.gpr .rdi = s.gpr .rdi + 4 ∧ - s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .r8 = s.gpr .r8 + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rdi, .rsi, .r8, .rcx] s s' := by - refine WP.keep _ ?_ (by decide) - unfold step3 - xrund [hw, List.cons_append, List.nil_append] - -/-- A body: a head that leaves `x` in `rax`, `reduce`, and the store. -/ -theorem body_ok {head : List Instr} {x : BitVec 64} (s : State) - (hh : WP isa (.block head) s fun s' => (s'.gpr .rax = x ∧ s'.mem = s.mem) ∧ Keep [.rax, .rdx, .r9] s s') - (hw : InRegions s.wr (s.gpr .rdi) 4) : - WP isa (.block (head ++ reduce ++ ([.store32 (at_ .rdi 0) .r10] : List Instr) ++ step3)) s fun s' => - (s'.mem = s.mem.writeW (s.gpr .rdi) (BitVec.setWidth 32 (redD x)) ∧ s'.gpr .rdi = s.gpr .rdi + 4 ∧ - s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .r8 = s.gpr .r8 + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ - Keep [.rax, .rdx, .r9, .r10, .r11, .rdi, .rsi, .r8, .rcx] s s' := by - rw [List.append_assoc, List.append_assoc, WP.block_append_iff] - refine WP.mono hh fun s1 ⟨⟨ha, hm1⟩, k1⟩ => ?_ - rw [WP.block_append_iff] - refine WP.mono (reduce_ok s1) fun s2 ⟨⟨hr, hm2⟩, k2⟩ => ?_ - have k12 := k1.trans k2 - refine WP.mono (mulTail_ok s2 (by rw [k12.2.2, k12.gpr (by decide)]; exact hw)) - fun s3 ⟨⟨hm3, hdi, hsi, h8, hcx, hz⟩, k3⟩ => ⟨?_, (k12.trans k3).mono (by decide)⟩ - rw [hm3, hdi, hsi, h8, hcx, hz, hr, hm2, ha, hm1, k12.gpr (r := .rdi) (by decide), k12.gpr (r := .rsi) (by decide), - k12.gpr (r := .r8) (by decide), k12.gpr (r := .rcx) (by decide)] - exact ⟨rfl, rfl, rfl, rfl, rfl, rfl⟩ - -theorem prod32_val {a b : BitVec 32} {x y : Zq} (ha : a.toNat = x.val) (hb : b.toNat = y.val) : - (BitVec.setWidth 32 (redD (prod32 a b))).toNat = (x * y).val := by - have e : (prod32 a b).toNat = x.val * y.val := by - rw [prod32, BitVec.toNat_ofNat, toNat_setWidth64, toNat_setWidth64, ha, hb] - exact Nat.mod_eq_of_lt (Nat.lt_of_lt_of_le (mul_lt_q2 x.isLt y.isLt) (by decide)) - rw [redD32_toNat, e, val_mul] - -theorem prod32_add_val {a b c : BitVec 32} {x y z : Zq} (ha : a.toNat = x.val) (hb : b.toNat = y.val) - (hc : c.toNat = z.val) : - (BitVec.setWidth 32 (redD (prod32 a b + BitVec.setWidth 64 c))).toNat = (z + x * y).val := by - have e : (prod32 a b + BitVec.setWidth 64 c).toNat = x.val * y.val + z.val := by - have := mul_lt_q2 x.isLt y.isLt - have := val_lt z - rw [BitVec.toNat_add, prod32, BitVec.toNat_ofNat, toNat_setWidth64, toNat_setWidth64, toNat_setWidth64, ha, - hb, hc] - omega - rw [redD32_toNat, e, val_add', val_mul, Nat.add_comm, Nat.add_mod_mod] +open VG.Proof.MlKem.X86_64 (Keep XOnly WP.keep writesOnly gprPreserved_of ifn xmm_setXmm wp_rcxLoop + add_ofNat_zero) +open VG.Impl.MlKem.X86_64 (xb xmov rcxLoop) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced) + +/-! ## Four products -/ + +/-- `2⁶⁴ mod q` in each doubleword, as the prologue leaves it in `xmm11`. -/ +def r2V : BitVec 128 := shufDwords ((0 : BitVec 64) ++ BitVec.setWidth 64 2365951#32) 0 + +theorem dword_r2V {i : Nat} (hi : i < 4) : dword r2V i = 2365951#32 := by + rcases cases4 hi with rfl | rfl | rfl | rfl <;> decide + +/-- What `mulCore` leaves in `xmm3` of the vectors of `f` and `g`. -/ +def mulV (x y : BitVec 128) : BitVec 128 := csubV (montV (montV x y (shufDwords y 0xF5)) r2V r2V) + +/-- What `mulAddCore` leaves in `xmm3`, with the vector of `h`. -/ +def mulAddV (x y z : BitVec 128) : BitVec 128 := csubV (XBinOp.eval .paddd (mulV x y) z) + +theorem mul_lane {x y : BitVec 128} {a b : Nat → Zq} (hx : DLanes x a) (hy : DLanes y b) {i : Nat} + (hi : i < 4) : (dword (mulV x y) i).toNat = (a i * b i).val := by + have hzo : ZOdd y (shufDwords y 0xF5) := fun j hj => by + rw [dword_shufDwords _ _ (by omega)] + rcases (by omega : j = 0 ∨ j = 1) with rfl | rfl <;> rfl + have hb1 : ∀ i < 4, (dword x i).toNat * (dword y i).toNat < q * 2 ^ 32 := fun i hi => by + rw [hx i hi, hy i hi] + exact Nat.lt_of_lt_of_le (Nat.mul_lt_mul_of_lt_of_le (val_lt (a i)) (Nat.le_of_lt (val_lt (b i))) + (by decide)) (by decide) + have m1 := fun i (hi : i < 4) => dword_montV hzo hb1 hi + have hb2 : ∀ i < 4, (dword (montV x y (shufDwords y 0xF5)) i).toNat * (dword r2V i).toNat < q * 2 ^ 32 := + fun i hi => by + rw [m1 i hi, dword_r2V hi] + have := mont_lt (hb1 i hi) + rw [q_eq] at this ⊢ + exact Nat.lt_of_lt_of_le (Nat.mul_lt_mul_of_lt_of_le this (Nat.le_refl 2365951) (by decide)) (by decide) + have hzo2 : ZOdd r2V r2V := fun j hj => by rw [dword_r2V (by omega), dword_r2V (by omega)] + rw [mulV, dword_csubV _ hi, csubL_toNat (by rw [dword_montV hzo2 hb2 hi]; exact mont_lt (hb2 i hi)), + dword_montV hzo2 hb2 hi, condSub_mont (hb2 i hi), m1 i hi, dword_r2V hi, + show (2365951#32).toNat = 2 ^ 64 % q from rfl, mont_mont_R2, hx i hi, hy i hi, val_mul] + +theorem mulAdd_lane {x y z : BitVec 128} {a b c : Nat → Zq} (hx : DLanes x a) (hy : DLanes y b) + (hz : DLanes z c) {i : Nat} (hi : i < 4) : (dword (mulAddV x y z) i).toNat = (c i + a i * b i).val := by + have hm := mul_lane hx hy hi + rw [mulAddV, dword_csubV _ hi, dword_paddd _ _ hi, addD_toNat (by rw [hm]; exact val_lt _) + (by rw [hz i hi]; exact val_lt _), hm, hz i hi, Nat.add_comm, ← val_add] + +theorem mulCore_ok {s : State} (hc : VConsts s) (h11 : s.xmm .xmm11 = r2V) : + WP isa (.block mulCore) s fun s' => + s'.xmm .xmm3 = mulV (s.xmm .xmm3) (s.xmm .xmm13) ∧ XOnly [.xmm12, .xmm3, .xmm2, .xmm4] s s' := by + simp only [mulCore, vmont, vredc, vcsub, vcadd, xmov, xb, List.cons_append, List.nil_append] + vrun [eval_movdqa] + rw [hc.q, hc.qinv, h11] + exact ⟨rfl, by xonly⟩ + +theorem mulAddCore_ok {s : State} (hc : VConsts s) (h11 : s.xmm .xmm11 = r2V) : + WP isa (.block mulAddCore) s fun s' => + s'.xmm .xmm3 = mulAddV (s.xmm .xmm3) (s.xmm .xmm13) (s.xmm .xmm5) ∧ + XOnly [.xmm12, .xmm3, .xmm2, .xmm4] s s' := by + simp only [mulAddCore, mulCore, vmont, vredc, vcsub, vcadd, xmov, xb, List.cons_append, List.nil_append] + vrun [eval_movdqa] + rw [hc.q, hc.qinv, h11] + exact ⟨rfl, by xonly⟩ + +theorem mulPro_ok (s : State) : + WP isa (.block mulPro) s fun s' => VConsts s' ∧ s'.xmm .xmm11 = r2V ∧ s'.xmm .xmm6 = s.xmm .xmm6 ∧ + Keep [.rax] s s' ∧ s'.mem = s.mem := by + simp only [mulPro, vconsts, List.cons_append, List.nil_append] + vrund + refine ⟨⟨?_, ?_⟩, ⟨fun r hr => ?_, rfl, rfl⟩⟩ + · simp only [RegUpd.xmm_setReg, xmm_setXmm, ite_true, ite_false, reduceCtorEq]; decide + · simp only [RegUpd.xmm_setReg, xmm_setXmm, ite_true, ite_false, reduceCtorEq]; decide + · simp only [List.mem_singleton] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setXmm, hr, ite_false] /-! ## The loop -/ namespace Mul -/-- After `i` coefficients, each one `v k`. -/ -structure Inv (s₀ : State) (v : Nat → BitVec 32) (i : Nat) (s : State) : Prop where - rdi : s.gpr .rdi = s₀.gpr .rdi + BitVec.ofNat 64 (4 * i) - rsi : s.gpr .rsi = s₀.gpr .rsi + BitVec.ofNat 64 (4 * i) - r8 : s.gpr .r8 = s₀.gpr .rdx + BitVec.ofNat 64 (4 * i) +theorem ptr16 (p : Addr) (i : Nat) : coeffAddr p (4 * i) + 16 = p + BitVec.ofNat 64 (16 * (i + 1)) := by + rw [coeffAddr, BitVec.add_assoc, show (16 : BitVec 64) = BitVec.ofNat 64 16 from rfl, ← BitVec.ofNat_add] + congr 2; omega + +/-- After `i` iterations: the first `4i` coefficients of `h` are `R`'s, the +others as they were in `s₀`. -/ +structure Inv (h f g : Addr) (s₀ : State) (R : Poly) (i : Nat) (s : State) : Prop where + rdi : s.gpr .rdi = h + BitVec.ofNat 64 (16 * i) + rsi : s.gpr .rsi = f + BitVec.ofNat 64 (16 * i) + rdx : s.gpr .rdx = g + BitVec.ofNat 64 (16 * i) rd : s.rd = s₀.rd wr : s.wr = s₀.wr - frame : Frame [pR (s₀.gpr .rdi)] s₀.mem s.mem - coeff : ∀ k < 256, coeffAt s.mem (s₀.gpr .rdi) k = if k < i then v k else coeffAt s₀.mem (s₀.gpr .rdi) k + c : VConsts s + r2 : s.xmm .xmm11 = r2V + x6 : s.xmm .xmm6 = s₀.xmm .xmm6 + frame : Frame [pR h] s₀.mem s.mem + done : ∀ k < 4 * i, (coeffAt s.mem h k).toNat = (R[k]!).val + rest : ∀ k < 256, 4 * i ≤ k → coeffAt s.mem h k = coeffAt s₀.mem h k section -variable {t : Poly → Poly → Poly → Poly} {r : Mem → Addr → Prop} {s₀ : State} (hp : (mulK t r).pre s₀) -include hp - -theorem inR {p : Addr} (hp' : p = s₀.gpr .rsi ∨ p = s₀.gpr .rdx ∨ p = s₀.gpr .rdi) {k : Nat} (hk : k < 256) - {s : State} (hrd : s.rd = s₀.rd) (hwr : s.wr = s₀.wr) : InRegions (s.rd ++ s.wr) (coeffAddr p k) 4 := by - rw [hrd, hwr, hp.1, hp.2.1] - rcases hp' with rfl | rfl | rfl - · exact ⟨_, by simp, coeff_contains _ hk⟩ - · exact ⟨_, by simp, coeff_contains _ hk⟩ - · exact ⟨_, by simp, coeff_contains _ hk⟩ - -theorem inW {k : Nat} (hk : k < 256) {s : State} (hwr : s.wr = s₀.wr) : - InRegions s.wr (coeffAddr (s₀.gpr .rdi) k) 4 := by - rw [hwr, hp.2.1]; exact ⟨_, by simp, coeff_contains _ hk⟩ - -/-- `f` and `g` are not written. -/ -theorem coeffF {m : Mem} (hf : Frame [pR (s₀.gpr .rdi)] s₀.mem m) {k : Nat} (hk : k < 256) : - coeffAt m (s₀.gpr .rsi) k = coeffAt s₀.mem (s₀.gpr .rsi) k := - coeffAt_frame hf (by simpa using hp.2.2.1.symm) hk - -theorem coeffG {m : Mem} (hf : Frame [pR (s₀.gpr .rdi)] s₀.mem m) {k : Nat} (hk : k < 256) : - coeffAt m (s₀.gpr .rdx) k = coeffAt s₀.mem (s₀.gpr .rdx) k := - coeffAt_frame hf (by simpa using hp.2.2.2.1.symm) hk - -omit hp in -theorem inv_step {v : Nat → BitVec 32} {i : Nat} (hi : i < 256) {s s' : State} (hI : Inv s₀ v i s) - (hm : s'.mem = s.mem.writeW (s.gpr .rdi) (v i)) (hdi : s'.gpr .rdi = s.gpr .rdi + 4) - (hsi : s'.gpr .rsi = s.gpr .rsi + 4) (h8 : s'.gpr .r8 = s.gpr .r8 + 4) (hrd : s'.rd = s.rd) - (hwr : s'.wr = s.wr) : Inv s₀ v (i + 1) s' where - rdi := by rw [hdi, hI.rdi]; exact ptr_step _ i 4 - rsi := by rw [hsi, hI.rsi]; exact ptr_step _ i 4 - r8 := by rw [h8, hI.r8]; exact ptr_step _ i 4 - rd := hrd.trans hI.rd - wr := hwr.trans hI.wr - frame := by - rw [hm, hI.rdi] - exact hI.frame.writeW (List.mem_singleton_self _) _ (coeff_contains _ hi) - coeff k hk := by - rw [hm, hI.rdi, ← coeffAddr, coeffAt_writeW _ _ hk hi, hI.coeff k hk] - by_cases e : i = k - · subst e; simp - · have : (k < i + 1) = (k < i) := propext (by omega) - simp only [e, this, ↓reduceIte] - -/-- What a body reads at coefficient `i`, and where. -/ -theorem reads {v : Nat → BitVec 32} {i : Nat} (hi : i < 256) {s : State} (hI : Inv s₀ v i s) : - s.mem.readW (s.gpr .rsi) 32 = coeffAt s₀.mem (s₀.gpr .rsi) i ∧ - s.mem.readW (s.gpr .r8) 32 = coeffAt s₀.mem (s₀.gpr .rdx) i ∧ - s.mem.readW (s.gpr .rdi) 32 = coeffAt s₀.mem (s₀.gpr .rdi) i ∧ - InRegions (s.rd ++ s.wr) (s.gpr .rsi) 4 ∧ InRegions (s.rd ++ s.wr) (s.gpr .r8) 4 ∧ - InRegions (s.rd ++ s.wr) (s.gpr .rdi) 4 ∧ InRegions s.wr (s.gpr .rdi) 4 := by - refine ⟨?_, ?_, ?_, ?_, ?_, ?_, ?_⟩ - · rw [hI.rsi, ← coeffAddr, ← coeffAt_eq, coeffF hp hI.frame hi] - · rw [hI.r8, ← coeffAddr, ← coeffAt_eq, coeffG hp hI.frame hi] - · rw [hI.rdi, ← coeffAddr, ← coeffAt_eq, hI.coeff i hi]; simp only [Nat.lt_irrefl, ↓reduceIte] - · rw [hI.rsi]; exact inR hp (.inl rfl) hi hI.rd hI.wr - · rw [hI.r8]; exact inR hp (.inr (.inl rfl)) hi hI.rd hI.wr - · rw [hI.rdi]; exact inR hp (.inr (.inr rfl)) hi hI.rd hI.wr - · rw [hI.rdi]; exact inW hp hi hI.wr - -/-- The whole function, from its precondition, with a body that stores -`v i` to coefficient `i`. -/ -theorem fn_ok {body : List Instr} {v : Nat → BitVec 32} - (hbody : ∀ i < 256, ∀ s, Inv s₀ v i s → WP isa (.block body) s fun s' => - (s'.mem = s.mem.writeW (s.gpr .rdi) (v i) ∧ s'.gpr .rdi = s.gpr .rdi + 4 ∧ - s'.gpr .rsi = s.gpr .rsi + 4 ∧ s'.gpr .r8 = s.gpr .r8 + 4 ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ - s'.zf = some (s.gpr .rcx - 1 == 0)) ∧ Keep [.rax, .rdx, .r9, .r10, .r11, .rdi, .rsi, .r8, .rcx] s s') - (hv : ∀ i < 256, (v i).toNat = - ((t (polyAt s₀.mem (s₀.gpr .rdi)) (polyAt s₀.mem (s₀.gpr .rsi)) (polyAt s₀.mem (s₀.gpr .rdx)))[i]!).val) - (hc : writesOnly [.rax, .rdx, .r9, .r10, .r11, .rdi, .rsi, .r8, .rcx] - (.seq (.block [.mov .r8 (.reg .rdx)]) (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block body) .ne))) = - true) - (hm : Code.allInstrs (fun i => !loadsMxcsr i) - (.seq (.block [.mov .r8 (.reg .rdx)]) (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block body) .ne)) : - Prog isa) = true) : - ∃ tr s', Exec isa (.seq (.block [.mov .r8 (.reg .rdx)]) - (.seq (.block [.mov32 .rcx (.imm 256)]) (.loop (.block body) .ne))) s₀ tr s' ∧ - abiPreserved s₀ s' ∧ (mulK t r).post s₀ s' := by - obtain ⟨tr, s', he, hI, hk⟩ := WP.keep [.rax, .rdx, .r9, .r10, .r11, .rdi, .rsi, .r8, .rcx] - (WP.seq (WP.mono (WP.keep [.r8] (Q := fun s => s.mem = s₀.mem ∧ s.gpr .r8 = s₀.gpr .rdx) (by xrund) - (by decide)) fun s1 ⟨⟨hm1, h81⟩, k1⟩ => - wp_counted (s₀ := s1) (N := 256) (v := 256) rfl (by decide) (Inv s₀ v) - (fun s hm hk' => ⟨by rw [hk'.gpr (by decide), k1.gpr (by decide)]; simp, - by rw [hk'.gpr (by decide), k1.gpr (by decide)]; simp, - by rw [hk'.gpr (by decide), h81]; simp, hk'.2.1.trans k1.2.1, hk'.2.2.trans k1.2.2, - by rw [hm, hm1]; exact Frame.refl _ _, fun k _ => by simp [hm, hm1]⟩) - fun i hi s hI => WP.mono (hbody i hi s hI) fun s' ⟨⟨hm, hdi, hsi, h8, hcx, hz⟩, hk⟩ => - ⟨inv_step hi hI hm hdi hsi h8 hk.2.1 hk.2.2, hcx, hz⟩)) hc - refine ⟨tr, s', he, abiPreserved_of_exec hm he (gprPreserved_of hk (by decide) hI.frame - (by simpa using hp.2.2.2.2.1)), polyIs_of_toNat fun i hi => ?_⟩ - rw [hI.coeff i hi, ifp hi] - exact hv i hi +variable {h f g : Addr} {s₀ : State} (hwh : pR h ∈ s₀.wr) (hrf : pR f ∈ s₀.rd ++ s₀.wr) + (hrg : pR g ∈ s₀.rd ++ s₀.wr) (hdf : (pR h).Disjoint (pR f)) (hdg : (pR h).Disjoint (pR g)) + {F G : Poly} (hF : ∀ k < 256, (coeffAt s₀.mem f k).toNat = (F[k]!).val) + (hG : ∀ k < 256, (coeffAt s₀.mem g k).toNat = (G[k]!).val) + {core : List Instr} {Fv : BitVec 128 → BitVec 128 → BitVec 128 → BitVec 128} + (hcore : ∀ s : State, VConsts s → s.xmm .xmm11 = r2V → WP isa (.block core) s fun s' => + s'.xmm .xmm3 = Fv (s.xmm .xmm3) (s.xmm .xmm13) (s.xmm .xmm5) ∧ XOnly [.xmm12, .xmm3, .xmm2, .xmm4] s s') + {R : Poly} {H : Nat → BitVec 32} (hH : ∀ k < 252, coeffAt s₀.mem h k = H k) + (hlane : ∀ i < 64, ∀ x y z : BitVec 128, (∀ e < 4, (dword x e).toNat = (F[4 * i + e]!).val) → + (∀ e < 4, (dword y e).toNat = (G[4 * i + e]!).val) → (∀ e < 4, dword z e = H (4 * i + e)) → + ∀ e < 4, (dword (Fv x y z) e).toNat = (R[4 * i + e]!).val) +include hwh hrf hrg hdf hdg hF hG hcore hH hlane + +theorem step {i : Nat} (hi : i < 63) {s : State} (hI : Inv h f g s₀ R i s) : + WP isa (.block (mulLoads ++ core ++ mulTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) s fun s' => + Inv h f g s₀ R (i + 1) s' ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by + have j0 : 4 * i + 4 ≤ 256 := by omega + have e1 : s.gpr .rdi = coeffAddr h (4 * i) := by rw [hI.rdi]; congr 2; omega + have e2 : s.gpr .rsi = coeffAddr f (4 * i) := by rw [hI.rsi]; congr 2; omega + have e3 : s.gpr .rdx = coeffAddr g (4 * i) := by rw [hI.rdx]; congr 2; omega + have rf : InRegions (s.rd ++ s.wr) (coeffAddr f (4 * i)) 16 := by + rw [hI.rd, hI.wr]; exact ⟨_, hrf, pR_contains f j0⟩ + have rg : InRegions (s.rd ++ s.wr) (coeffAddr g (4 * i)) 16 := by + rw [hI.rd, hI.wr]; exact ⟨_, hrg, pR_contains g j0⟩ + have rh : InRegions (s.rd ++ s.wr) (coeffAddr h (4 * i)) 16 := by + rw [hI.rd, hI.wr]; exact f_in (List.mem_append_right _ hwh) j0 + have wh : InRegions s.wr (coeffAddr h (4 * i)) 16 := by rw [hI.wr]; exact f_in hwh j0 + have mf : ∀ k < 256, coeffAt s.mem f k = coeffAt s₀.mem f k := fun k hk => + coeffAt_frame hI.frame (by simpa using hdf.symm) (by rw [n_eq]; exact hk) + have mg : ∀ k < 256, coeffAt s.mem g k = coeffAt s₀.mem g k := fun k hk => + coeffAt_frame hI.frame (by simpa using hdg.symm) (by rw [n_eq]; exact hk) + rw [show mulLoads ++ core ++ mulTail ++ [.alu .sub .rcx (.imm 1)] = + mulLoads ++ (core ++ (mulTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) by simp [List.append_assoc], + WP.block_append_iff] + simp only [mulLoads] + vrund [e1, e2, e3, rf, rg, rh] + rw [WP.block_append_iff] + refine WP.mono (hcore _ (((hI.c.setXmm (by decide) (by decide) _).setXmm (by decide) (by decide) _).setXmm + (by decide) (by decide) _) (by simp only [xmm_setXmm, reduceCtorEq, ite_false]; exact hI.r2)) + fun s2 ⟨h3, o2⟩ => ?_ + have c2 := xonly_vconsts o2 (((hI.c.setXmm (by decide) (by decide) _).setXmm (by decide) (by decide) _).setXmm + (by decide) (by decide) _) (by decide) (by decide) + have g2 : s2.gpr = s.gpr := o2.gpr + have m2 : s2.mem = s.mem := o2.mem + have r2 : s2.rd = s.rd := o2.rd + have w2 : s2.wr = s.wr := o2.wr + have x11 : s2.xmm .xmm11 = r2V := by rw [o2.xmm _ (by decide)]; simp only [xmm_setXmm, reduceCtorEq, ite_false]; exact hI.r2 + simp only [xmm_setXmm, ite_true, reduceCtorEq, ite_false] at h3 + generalize hV : Fv (s.mem.readW (coeffAddr f (4 * i)) 128) (s.mem.readW (coeffAddr g (4 * i)) 128) + (s.mem.readW (coeffAddr h (4 * i)) 128) = V at h3 + simp only [mulTail] + vrund [g2, m2, r2, w2, e1, e2, e3, wh, h3] + have lx : ∀ e < 4, (dword (s.mem.readW (coeffAddr f (4 * i)) 128) e).toNat = (F[4 * i + e]!).val := + fun e he => by rw [dword_readW _ _ he, coeffAddr_add, ← coeffAt_eq, mf _ (by omega), hF _ (by omega)] + have ly : ∀ e < 4, (dword (s.mem.readW (coeffAddr g (4 * i)) 128) e).toNat = (G[4 * i + e]!).val := + fun e he => by rw [dword_readW _ _ he, coeffAddr_add, ← coeffAt_eq, mg _ (by omega), hG _ (by omega)] + have lz : ∀ e < 4, dword (s.mem.readW (coeffAddr h (4 * i)) 128) e = H (4 * i + e) := + fun e he => by rw [dword_readW _ _ he, coeffAddr_add, ← coeffAt_eq, hI.rest _ (by omega) (by omega), + hH _ (by omega)] + have hl := hlane i (by omega) _ _ _ lx ly lz + rw [hV] at hl + refine ⟨?_, ?_, ?_, ?_, ?_, ⟨?_, ?_⟩, ?_, ?_, ?_, ?_, ?_⟩ <;> + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, RegUpd.mem_setReg, RegUpd.mem_setFlags, + RegUpd.rd_setReg, RegUpd.rd_setFlags, RegUpd.wr_setReg, RegUpd.wr_setFlags, RegUpd.xmm_setReg, + RegUpd.xmm_setFlags, ite_true, ite_false, reduceCtorEq] + · exact ptr16 h i + · exact ptr16 f i + · exact ptr16 g i + · exact hI.rd + · exact hI.wr + · exact c2.q + · exact c2.qinv + · exact x11 + · rw [o2.xmm _ (by decide)]; simp only [xmm_setXmm, reduceCtorEq, ite_false]; exact hI.x6 + · exact hI.frame.writeW (List.mem_singleton_self _) _ (pR_contains h j0) + · intro k hk + rw [coeffAt_write128 _ _ j0 _ (by omega)] + split + · rw [hl _ (by omega), show 4 * i + (k - 4 * i) = k by omega] + · exact hI.done k (by omega) + · intro k hk hk' + rw [coeffAt_write128 _ _ j0 _ hk, ifn (by omega)] + exact hI.rest k hk (by omega) + +theorem loop_ok (hdi : s₀.gpr .rdi = h) (hsi : s₀.gpr .rsi = f) (hdx : s₀.gpr .rdx = g) (hc : VConsts s₀) + (h11 : s₀.xmm .xmm11 = r2V) : WP isa (rcxLoop 63 (mulLoads ++ core ++ mulTail)) s₀ (Inv h f g s₀ R 63) := + wp_rcxLoop (N := 63) (by decide) (by decide) _ (fun u o _ => + ⟨by rw [o.keep.gpr (by decide), hdi, Nat.mul_zero, add_ofNat_zero], + by rw [o.keep.gpr (by decide), hsi, Nat.mul_zero, add_ofNat_zero], + by rw [o.keep.gpr (by decide), hdx, Nat.mul_zero, add_ofNat_zero], o.keep.2.1, o.keep.2.2, + ⟨by rw [o.xmm]; exact hc.q, by rw [o.xmm]; exact hc.qinv⟩, by rw [o.xmm]; exact h11, by rw [o.xmm], + by rw [o.mem]; exact Frame.refl _ _, fun k hk => absurd hk (by omega), fun k _ _ => by rw [o.mem]⟩) + fun i hi u hI => step hwh hrf hrg hdf hdg hF hG hcore hH hlane hi hI + + +omit hwh hH in +/-- The last four coefficients, with those of `h` in `xmm6`. -/ +theorem last {s : State} (hI : Inv h f g s₀ R 63 s) (hz : ∀ e < 4, dword (s₀.xmm .xmm6) e = H (252 + e)) : + WP isa (.block (mulLast core)) s fun s' => + (∀ e < 4, (dword (s'.xmm .xmm3) e).toNat = (R[252 + e]!).val) ∧ s'.gpr = s.gpr ∧ s'.mem = s.mem := by + have j0 : 4 * 63 + 4 ≤ 256 := by decide + have e2 : s.gpr .rsi = coeffAddr f (4 * 63) := hI.rsi + have e3 : s.gpr .rdx = coeffAddr g (4 * 63) := hI.rdx + have rf : InRegions (s.rd ++ s.wr) (coeffAddr f (4 * 63)) 16 := by + rw [hI.rd, hI.wr]; exact ⟨_, hrf, pR_contains f j0⟩ + have rg : InRegions (s.rd ++ s.wr) (coeffAddr g (4 * 63)) 16 := by + rw [hI.rd, hI.wr]; exact ⟨_, hrg, pR_contains g j0⟩ + have mf : ∀ k < 256, coeffAt s.mem f k = coeffAt s₀.mem f k := fun k hk => + coeffAt_frame hI.frame (by simpa using hdf.symm) (by rw [n_eq]; exact hk) + have mg : ∀ k < 256, coeffAt s.mem g k = coeffAt s₀.mem g k := fun k hk => + coeffAt_frame hI.frame (by simpa using hdg.symm) (by rw [n_eq]; exact hk) + rw [mulLast, WP.block_append_iff] + vrund [e2, e3, rf, rg, eval_movdqa] + refine WP.mono (hcore _ (((hI.c.setXmm (by decide) (by decide) _).setXmm (by decide) (by decide) _).setXmm + (by decide) (by decide) _) (by simp only [xmm_setXmm, reduceCtorEq, ite_false]; exact hI.r2)) + fun s2 ⟨h3, o2⟩ => ⟨fun e he => ?_, o2.gpr, o2.mem⟩ + simp only [xmm_setXmm, ite_true, reduceCtorEq, ite_false, hI.x6] at h3 + rw [h3] + refine hlane 63 (by decide) _ _ _ (fun e he => ?_) (fun e he => ?_) hz e he + · rw [dword_readW _ _ he, coeffAddr_add, ← coeffAt_eq, mf _ (by omega), hF _ (by omega)] + · rw [dword_readW _ _ he, coeffAddr_add, ← coeffAt_eq, mg _ (by omega), hG _ (by omega)] end + +theorem coeffAt_mxH {m m' : Mem} {p : Addr} (hf : Frame [mxH p] m m') {k : Nat} (hk : k < 254) : + coeffAt m' p k = coeffAt m p k := + hf.readW (r := ⟨coeffAddr p k, 4⟩) (Region.contains_self _ _) (fun r hr => by + simp only [List.mem_singleton] at hr; subst hr + exact Offset.disjoint p (by omega) (by omega) (by omega)) (by decide) + +/-- The whole function, from its precondition, with a `core` whose lanes are +those of `t`. -/ +theorem fn_ok {t : Poly → Poly → Poly → Poly} {hPre : Mem → Addr → Prop} {σ : State} (hp : (mulK t hPre).pre σ) + {core : List Instr} {Fv : BitVec 128 → BitVec 128 → BitVec 128 → BitVec 128} + (hcore : ∀ s : State, VConsts s → s.xmm .xmm11 = r2V → WP isa (.block core) s fun s' => + s'.xmm .xmm3 = Fv (s.xmm .xmm3) (s.xmm .xmm13) (s.xmm .xmm5) ∧ XOnly [.xmm12, .xmm3, .xmm2, .xmm4] s s') + (hlane : ∀ i < 64, ∀ x y z : BitVec 128, + (∀ e < 4, (dword x e).toNat = ((polyAt σ.mem (σ.gpr .rsi))[4 * i + e]!).val) → + (∀ e < 4, (dword y e).toNat = ((polyAt σ.mem (σ.gpr .rdx))[4 * i + e]!).val) → + (∀ e < 4, dword z e = coeffAt σ.mem (σ.gpr .rdi) (4 * i + e)) → + ∀ e < 4, (dword (Fv x y z) e).toNat = ((t (polyAt σ.mem (σ.gpr .rdi)) (polyAt σ.mem (σ.gpr .rsi)) + (polyAt σ.mem (σ.gpr .rdx)))[4 * i + e]!).val) + (hk : writesOnly [.rax, .rdi, .rsi, .rdx, .rcx] + (.seq (.block mulPro) (.seq (rcxLoop 63 (mulLoads ++ core ++ mulTail)) (.block (mulLast core)))) = true) : + WP isa (mulFn core) σ fun s' => Keep [.r8, .rax, .r11, .rax, .rdi, .rsi, .rdx, .rcx] σ s' ∧ + Frame [pR (σ.gpr .rdi)] σ.mem s'.mem ∧ (mulK t hPre).post σ s' := by + obtain ⟨hrd, hwr, hdf, hdg, -, -, -, -, redf, redg⟩ := hp + generalize eh : σ.gpr .rdi = h at * + generalize ef : σ.gpr .rsi = f at * + generalize eg : σ.gpr .rdx = g at * + let R := t (polyAt σ.mem h) (polyAt σ.mem f) (polyAt σ.mem g) + have hwh : pR h ∈ σ.wr := by rw [hwr]; exact List.mem_singleton_self _ + have r6 : InRegions (σ.rd ++ σ.wr) (coeffAddr h 252) 16 := + ⟨_, List.mem_append_right _ hwh, Offset.contains_base h (by omega) (by omega)⟩ + simp only [mulFn] + refine WP.seq (WP.mono (Q := fun (s1 : State) => s1.gpr .r8 = h ∧ + s1.xmm .xmm6 = σ.mem.readW (coeffAddr h 252) 128 ∧ Keep [.r8] σ s1 ∧ s1.mem = σ.mem) (by + vrund [eh, r6] + exact ⟨fun r hr => by + simp only [List.mem_singleton] at hr + simp only [RegUpd.gpr_setXmm, RegUpd.gpr_setReg, hr, ite_false], rfl, rfl⟩) fun s1 ⟨h8, h6, k1, m1⟩ => ?_) + have hw1 : pR h ∈ s1.wr := by rw [k1.2.2]; exact hwh + refine WP.seq (WP.mono (withMxcsrH_ok (r := .r8) ⟨by decide, by decide⟩ [.rax, .rdi, .rsi, .rdx, .rcx] + ⟨by decide, by decide⟩ h8 hw1 hk (Q := fun (s3 : State) => Keep [.rax, .r11, .rax, .rdi, .rsi, .rdx, .rcx] s1 s3 ∧ + s3.gpr .rdi = coeffAddr h 252 ∧ Frame [pR h] σ.mem s3.mem ∧ + (∀ k < 252, (coeffAt s3.mem h k).toNat = (R[k]!).val) ∧ + ∀ e < 4, (dword (s3.xmm .xmm3) e).toNat = (R[252 + e]!).val) + fun s2 k2 f2 x2 => ?_) fun s4 ⟨s3, ⟨kk, hdi, fr, dn, ln⟩, f4, k4, x4⟩ => ?_) + · have fσ2 : Frame [mxH h] σ.mem s2.mem := by rw [← m1]; exact f2 + refine WP.mono (WP.keep [.rax, .rdi, .rsi, .rdx, .rcx] (WP.seq (WP.mono (mulPro_ok s2) + fun w ⟨cw, xw, x6, kw, mw⟩ => ?_) (Q := fun (s3 : State) => s3.gpr .rdi = coeffAddr h 252 ∧ + Frame [pR h] σ.mem s3.mem ∧ (∀ k < 252, (coeffAt s3.mem h k).toNat = (R[k]!).val) ∧ + ∀ e < 4, (dword (s3.xmm .xmm3) e).toNat = (R[252 + e]!).val)) hk) + fun s3 ⟨q3, kk⟩ => ⟨k2.trans kk, q3⟩ + have gw : ∀ r, r ≠ .rax → r ≠ .r11 → r ≠ .r8 → w.gpr r = σ.gpr r := fun r h1 h2 h3 => by + rw [kw.gpr (by simpa using h1), k2.gpr (by simp [h1, h2]), k1.gpr (by simpa using h3)] + have rw' : w.rd = σ.rd ∧ w.wr = σ.wr := + ⟨kw.2.1.trans (k2.2.1.trans k1.2.1), kw.2.2.trans (k2.2.2.trans k1.2.2)⟩ + have fσw : Frame [mxH h] σ.mem w.mem := by rw [mw]; exact fσ2 + have fσw' : Frame [pR h] σ.mem w.mem := + Frame.sub fσw fun r hr => ⟨pR h, List.mem_singleton_self _, by + simp only [List.mem_singleton] at hr; subst hr; exact mxH_sub h⟩ + refine WP.seq (WP.mono (loop_ok (s₀ := w) (h := h) (f := f) (g := g) (R := R) (F := polyAt σ.mem f) + (G := polyAt σ.mem g) (H := coeffAt σ.mem h) + (by rw [rw'.2]; exact hwh) (by rw [rw'.1, hrd]; simp) (by rw [rw'.1, hrd]; simp) hdf hdg + (fun k hk => by + rw [coeffAt_frame fσw' (by simpa using hdf.symm) (by rw [n_eq]; exact hk), + polyAt_val redf (by rw [n_eq]; exact hk)]) + (fun k hk => by + rw [coeffAt_frame fσw' (by simpa using hdg.symm) (by rw [n_eq]; exact hk), + polyAt_val redg (by rw [n_eq]; exact hk)]) + hcore (fun k hk => coeffAt_mxH fσw (by omega)) hlane + (by rw [gw _ (by decide) (by decide) (by decide), eh]) (by rw [gw _ (by decide) (by decide) (by decide), ef]) + (by rw [gw _ (by decide) (by decide) (by decide), eg]) cw xw) fun s hI => ?_) + refine WP.mono (last (s₀ := w) (h := h) (f := f) (g := g) (R := R) (F := polyAt σ.mem f) + (G := polyAt σ.mem g) (H := coeffAt σ.mem h) + (by rw [rw'.1, hrd]; simp) (by rw [rw'.1, hrd]; simp) hdf hdg + (fun k hk => by + rw [coeffAt_frame fσw' (by simpa using hdf.symm) (by rw [n_eq]; exact hk), + polyAt_val redf (by rw [n_eq]; exact hk)]) + (fun k hk => by + rw [coeffAt_frame fσw' (by simpa using hdg.symm) (by rw [n_eq]; exact hk), + polyAt_val redg (by rw [n_eq]; exact hk)]) + hcore hlane hI (fun e he => by + rw [x6, x2, h6, dword_readW _ _ he, coeffAddr_add, ← coeffAt_eq])) fun s3 ⟨ln, g3, m3⟩ => ?_ + refine ⟨by rw [g3, hI.rdi], ?_, fun k hk => by rw [m3]; exact hI.done k (by omega), ln⟩ + rw [m3] + exact Frame.trans fσw' hI.frame + · have k14 := (k1.trans kk).trans k4 + have hdi4 : s4.gpr .rdi = coeffAddr h 252 := by rw [k4.gpr (by simp), hdi] + have wh4 : InRegions s4.wr (coeffAddr h 252) 16 := by + rw [k14.2.2]; exact ⟨_, hwh, Offset.contains_base h (by omega) (by omega)⟩ + vrund [hdi4, wh4] + have f4' : Frame [pR h] s3.mem s4.mem := Frame.sub f4 fun r hr => ⟨pR h, List.mem_singleton_self _, by + simp only [List.mem_singleton] at hr; subst hr; exact mxH_sub h⟩ + refine ⟨k14.mono (by simp), (fr.trans f4').writeW (List.mem_singleton_self _) _ + (Offset.contains_base h (by omega) (by omega)), ?_⟩ + dsimp only [mulK] + rw [eh, ef, eg] + refine polyIs_of_toNat fun k hk => ?_ + rw [n_eq] at hk + rw [coeffAt_write128 _ _ (j := 252) (by decide) _ hk] + split + · rw [x4] + have := ln (k - 252) (by omega) + rwa [show 252 + (k - 252) = k by omega] at this + · rw [coeffAt_mxH f4 (by omega)] + exact dn k (by omega) + end Mul /-! ## The functions -/ @@ -211,31 +359,27 @@ abbrev mulK' : Contract isa := mulK (fun _ f g => Spec.MlDsa.multiplyNTT f g) fu abbrev mulAddK : Contract isa := mulK (fun h f g => Spec.MlDsa.add h (Spec.MlDsa.multiplyNTT f g)) Reduced theorem mul_correct (s : State) (hs : mulK'.pre s) : - ∃ t s', Exec isa Impl.MlDsa.X86_64.Arith.mul s t s' ∧ abiPreserved s s' ∧ mulK'.post s s' := - Mul.fn_ok hs (v := fun i => BitVec.setWidth 32 (redD (prod32 (coeffAt s.mem (s.gpr .rsi) i) - (coeffAt s.mem (s.gpr .rdx) i)))) - (fun i hi s' hI => by - obtain ⟨e1, e2, _, h1, h2, _, hw⟩ := Mul.reads hs hi hI - have := body_ok s' (mulHead_ok s' h1 h2) hw - rwa [e1, e2] at this) - (fun i hi => by - rw [mul_get _ _ hi] - exact prod32_val (polyAt_val hs.2.2.2.2.2.2.2.2.1 hi).symm (polyAt_val hs.2.2.2.2.2.2.2.2.2 hi).symm) - (by decide) (by decide) + ∃ t s', Exec isa Impl.MlDsa.X86_64.Arith.mul s t s' ∧ abiPreserved s s' ∧ mulK'.post s s' := by + obtain ⟨t, s', he, hk, hf, hq⟩ := Mul.fn_ok hs (core := mulCore) (Fv := fun x y _ => mulV x y) + (fun s hc h11 => mulCore_ok hc h11) + (fun i hi x y z hx hy _ e he => by + rw [mul_get _ _ (by rw [n_eq]; omega)] + exact mul_lane hx hy he) + (by decide +kernel) + exact ⟨t, s', he, abiPreserved_of_ctl (by decide +kernel) he (gprPreserved_of hk (by decide) hf + (by simpa using hs.2.2.2.2.1)), hq⟩ theorem mulAdd_correct (s : State) (hs : mulAddK.pre s) : - ∃ t s', Exec isa Impl.MlDsa.X86_64.Arith.mulAdd s t s' ∧ abiPreserved s s' ∧ mulAddK.post s s' := - Mul.fn_ok hs (v := fun i => BitVec.setWidth 32 (redD (prod32 (coeffAt s.mem (s.gpr .rsi) i) - (coeffAt s.mem (s.gpr .rdx) i) + BitVec.setWidth 64 (coeffAt s.mem (s.gpr .rdi) i)))) - (fun i hi s' hI => by - obtain ⟨e1, e2, e3, h1, h2, h3, hw⟩ := Mul.reads hs hi hI - have := body_ok s' (mulAddHead_ok s' h1 h2 h3) hw - rwa [e1, e2, e3] at this) - (fun i hi => by - rw [add_get _ _ hi, mul_get _ _ hi] - exact prod32_add_val (polyAt_val hs.2.2.2.2.2.2.2.2.1 hi).symm - (polyAt_val hs.2.2.2.2.2.2.2.2.2 hi).symm (polyAt_val hs.2.2.2.2.2.2.2.1 hi).symm) - (by decide) (by decide) + ∃ t s', Exec isa Impl.MlDsa.X86_64.Arith.mulAdd s t s' ∧ abiPreserved s s' ∧ mulAddK.post s s' := by + obtain ⟨t, s', he, hk, hf, hq⟩ := Mul.fn_ok hs (core := mulAddCore) (Fv := mulAddV) + (fun s hc h11 => mulAddCore_ok hc h11) + (fun i hi x y z hx hy hz e he => by + rw [add_get _ _ (by rw [n_eq]; omega), mul_get _ _ (by rw [n_eq]; omega)] + exact mulAdd_lane hx hy (c := fun e => (polyAt s.mem (s.gpr .rdi))[4 * i + e]!) + (fun e he => by rw [hz e he, polyAt_val hs.2.2.2.2.2.2.2.1 (by rw [n_eq]; omega)]) he) + (by decide +kernel) + exact ⟨t, s', he, abiPreserved_of_ctl (by decide +kernel) he (gprPreserved_of hk (by decide) hf + (by simpa using hs.2.2.2.2.1)), hq⟩ /-- The pointers and `rsp` are public. -/ def mulτ : X86_64.Taint.T := X86_64.Taint.ofRegs [.rdi, .rsi, .rdx, .rsp] diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean new file mode 100644 index 000000000..d87f9e044 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean @@ -0,0 +1,73 @@ +import VerifiedGarbage.Proof.MlKem.X86_64.VMxcsr +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Basic + +/-! +# ML-DSA on x86-64: MXCSR through the end of a polynomial + +Untrusted: everything here is checked by Lean. `withMxcsr r 1016 c` +(ML-KEM's, see `Impl/MlKem/X86_64/Vec.lean`) through the last 8 bytes `mxH` +of a writable polynomial at `r` (`withMxcsrH_ok`), as ML-KEM's +`withMxcsr_ok` through `scratch + 768`. +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 +open VG.Proof.MlKem.X86_64 (Keep WP.keep writesOnly ldmxcsr_ok) + +/-- The last 8 bytes of the polynomial at `p`. -/ +abbrev mxH (p : Addr) : Region := ⟨p + BitVec.ofNat 64 1016, 8⟩ + +theorem mxH_in {p : Addr} {rs : List Region} (hw : pR p ∈ rs) (d : Nat) (hd : 1016 ≤ d ∧ d ≤ 1020) : + InRegions rs (p + BitVec.ofNat 64 d) 4 := + ⟨_, hw, Offset.contains_base p (by omega) (by omega)⟩ + +theorem mxH_sub (p : Addr) : Region.Sub (mxH p) (pR p) := Offset.sub_base p (by decide) + +/-- `withMxcsr` through `mxH`: `c` runs from `s` but for `rax`, `r11` and +those bytes, and nothing more than they and MXCSR change after it. -/ +theorem withMxcsrH_ok {c : Prog isa} {r : Reg} (hr : r ≠ .r11 ∧ r ≠ .rax) (rs : List Reg) + (hrs : r ∉ rs ∧ Reg.r11 ∉ rs) {p : Addr} {s : State} {Q : State → Prop} + (hsi : s.gpr r = p) (hw : pR p ∈ s.wr) (hk : writesOnly rs c = true) + (hc : ∀ s1, Keep [.rax, .r11] s s1 → Frame [mxH p] s.mem s1.mem → s1.xmm = s.xmm → WP isa c s1 Q) : + WP isa (withMxcsr r 1016 c) s fun s' => + ∃ s2, Q s2 ∧ Frame [mxH p] s2.mem s'.mem ∧ Keep [] s2 s' ∧ s'.xmm = s2.xmm := by + have h0 := mxH_in hw 1016 (by decide) + have h0' := mxH_in (List.mem_append_right s.rd hw) 1016 (by decide) + have h4 := mxH_in hw 1020 (by decide) + simp only [withMxcsr] + refine WP.seq (WP.mono (Q := fun (s1 : State) => s1.gpr .r11 = (s.mxcsr &&& 0xFFFF).setWidth 64 ∧ Keep [.r11] s s1 ∧ + Frame [mxH p] s.mem s1.mem ∧ s1.xmm = s.xmm) (by + vrunm [hsi, h0, h0', Mem.readW_writeW_self32, hr.1] + refine ⟨by rw [BitVec.setWidth_setWidth_of_le _ (by decide), BitVec.setWidth_eq], + ⟨fun r hr => ?_, rfl, rfl⟩, (Frame.refl _ _).writeW (List.mem_singleton_self _) _ + (Offset.contains p (by decide) (by decide) (by decide))⟩ + simp only [List.mem_singleton] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false]) fun s1 ⟨h11, k1, f1, x1⟩ => ?_) + have hsi1 : s1.gpr r = p := by rw [k1.gpr (by simpa using hr.1), hsi] + have h4' : InRegions s1.wr (p + BitVec.ofNat 64 (1016 + 4)) 4 := by rw [k1.2.2]; exact h4 + have h4'' : InRegions (s1.rd ++ s1.wr) (p + BitVec.ofNat 64 (1016 + 4)) 4 := + let ⟨r, hr, hc⟩ := h4'; ⟨r, List.mem_append_right _ hr, hc⟩ + refine WP.seq (WP.seq (WP.mono (Q := fun (s2 : State) => Keep [.rax] s1 s2 ∧ Frame [mxH p] s1.mem s2.mem ∧ + s2.xmm = s1.xmm) + (by + vrunm [hsi1, h4', h4'', Mem.readW_writeW_self32, hr.2] + refine ⟨⟨fun r hr => ?_, rfl, rfl⟩, (Frame.refl _ _).writeW (List.mem_singleton_self _) _ + (Offset.contains p (by decide) (by decide) (by decide))⟩ + simp only [List.mem_singleton] at hr + simp only [RegUpd.gpr_setReg, hr, ite_false]) fun s2 ⟨k2, f2, x2⟩ => ?_)) + refine WP.seq (WP.mono (WP.keep _ (hc s2 ((k1.trans k2).mono (by simp)) (f1.trans f2) (x2.trans x1)) hk) + fun s3 ⟨hq, k3⟩ => ?_) + have k23 := k2.trans k3 + have hsi3 : s3.gpr r = p := by rw [k23.gpr (by simp [hr.2, hrs.1]), hsi1] + have h113 : s3.gpr .r11 = BitVec.setWidth 64 (s.mxcsr &&& 65535) := by rw [k23.gpr (by simp [hrs.2]), h11] + have h03 : InRegions s3.wr (p + BitVec.ofNat 64 1016) 4 := by rw [k23.2.2, k1.2.2]; exact h0 + have h03' : InRegions (s3.rd ++ s3.wr) (p + BitVec.ofNat 64 1016) 4 := + let ⟨r, hr, hc⟩ := h03; ⟨r, List.mem_append_right _ hr, hc⟩ + refine WP.mono (Q := fun s4 => s4 = s3) (by vrunm) fun s4 h4 => ?_ + subst h4 + vrunm [hsi3, h113, h03, h03', Mem.readW_writeW_self32, ldmxcsr_ok] + exact ⟨_, hq, (Frame.refl _ _).writeW (List.mem_singleton_self _) _ + (Offset.contains p (by decide) (by decide) (by decide)), ⟨fun _ _ => rfl, rfl, rfl⟩, rfl⟩ + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/src/asm/x86_64/mldsa.rs b/src/asm/x86_64/mldsa.rs index 59b6ad833..01e804f36 100644 --- a/src/asm/x86_64/mldsa.rs +++ b/src/asm/x86_64/mldsa.rs @@ -1424,6 +1424,8 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_inv_ntt(f: *mut [u32; 256], scratc /// /// Contract: `VG.Spec.MlDsa.mulContract`. Constant time: only the pointers may affect timing, not the data. /// +/// The function computes on four coefficients at a time in SSE2 registers. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before returning. +/// /// # Safety /// /// * `h` must be valid for reads and writes of 1024 bytes. @@ -1436,29 +1438,106 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_inv_ntt(f: *mut [u32; 256], scratc #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_ntt(h: *mut [u32; 256], f: *const [u32; 256], g: *const [u32; 256]) { core::arch::naked_asm!( - "mov r8, rdx", - "mov ecx, 256", + "mov r8, rdi", + "movdqu xmm6, XMMWORD PTR [rdi+1008]", + "stmxcsr DWORD PTR [r8+1016]", + "mov r11d, DWORD PTR [r8+1016]", + "and r11d, 65535", + "mov eax, 8127", + "mov DWORD PTR [r8+1020], eax", + "ldmxcsr DWORD PTR [r8+1020]", + "lfence", + "mov eax, 8380417", + "movq xmm15, rax", + "pshufd xmm15, xmm15, 0", + "mov eax, -58728449", + "movq xmm14, rax", + "pshufd xmm14, xmm14, 0", + "mov eax, 2365951", + "movq xmm11, rax", + "pshufd xmm11, xmm11, 0", + "mov ecx, 63", "20:", - "mov eax, DWORD PTR [rsi]", - "mov r9d, DWORD PTR [r8]", - "mul r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rdi], r10d", - "add rdi, 4", - "add rsi, 4", - "add r8, 4", + "movdqu xmm3, XMMWORD PTR [rsi]", + "movdqu xmm13, XMMWORD PTR [rdx]", + "movdqu xmm5, XMMWORD PTR [rdi]", + "pshufd xmm12, xmm13, 245", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm11", + "pmuludq xmm4, xmm11", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdi], xmm3", + "add rdi, 16", + "add rsi, 16", + "add rdx, 16", "sub rcx, 1", "jne 20b", + "movdqu xmm3, XMMWORD PTR [rsi]", + "movdqu xmm13, XMMWORD PTR [rdx]", + "movdqa xmm5, xmm6", + "pshufd xmm12, xmm13, 245", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm11", + "pmuludq xmm4, xmm11", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "lfence", + "mov DWORD PTR [r8+1016], r11d", + "ldmxcsr DWORD PTR [r8+1016]", + "movdqu XMMWORD PTR [rdi], xmm3", "ret", ) } @@ -1467,6 +1546,8 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_ntt(h: *mut [u32; 256], f /// /// Contract: `VG.Spec.MlDsa.mulAddContract`. Constant time: only the pointers may affect timing, not the data. /// +/// The function computes on four coefficients at a time in SSE2 registers. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before returning. +/// /// # Safety /// /// * `h` must be valid for reads and writes of 1024 bytes. @@ -1480,31 +1561,118 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_ntt(h: *mut [u32; 256], f #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_add_ntt(h: *mut [u32; 256], f: *const [u32; 256], g: *const [u32; 256]) { core::arch::naked_asm!( - "mov r8, rdx", - "mov ecx, 256", + "mov r8, rdi", + "movdqu xmm6, XMMWORD PTR [rdi+1008]", + "stmxcsr DWORD PTR [r8+1016]", + "mov r11d, DWORD PTR [r8+1016]", + "and r11d, 65535", + "mov eax, 8127", + "mov DWORD PTR [r8+1020], eax", + "ldmxcsr DWORD PTR [r8+1020]", + "lfence", + "mov eax, 8380417", + "movq xmm15, rax", + "pshufd xmm15, xmm15, 0", + "mov eax, -58728449", + "movq xmm14, rax", + "pshufd xmm14, xmm14, 0", + "mov eax, 2365951", + "movq xmm11, rax", + "pshufd xmm11, xmm11, 0", + "mov ecx, 63", "20:", - "mov eax, DWORD PTR [rsi]", - "mov r9d, DWORD PTR [r8]", - "mul r9", - "mov r9d, DWORD PTR [rdi]", - "add rax, r9", - "mov r10, rax", - "movabs r11, 2201172575745", - "mul r11", - "mov rax, rdx", - "mov r11, 8380417", - "mul r11", - "sub r10, rax", - "sub r10d, 8380417", - "sbb r11d, r11d", - "and r11d, 8380417", - "add r10d, r11d", - "mov DWORD PTR [rdi], r10d", - "add rdi, 4", - "add rsi, 4", - "add r8, 4", + "movdqu xmm3, XMMWORD PTR [rsi]", + "movdqu xmm13, XMMWORD PTR [rdx]", + "movdqu xmm5, XMMWORD PTR [rdi]", + "pshufd xmm12, xmm13, 245", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm11", + "pmuludq xmm4, xmm11", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "paddd xmm3, xmm5", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "movdqu XMMWORD PTR [rdi], xmm3", + "add rdi, 16", + "add rsi, 16", + "add rdx, 16", "sub rcx, 1", "jne 20b", + "movdqu xmm3, XMMWORD PTR [rsi]", + "movdqu xmm13, XMMWORD PTR [rdx]", + "movdqa xmm5, xmm6", + "pshufd xmm12, xmm13, 245", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm13", + "pmuludq xmm4, xmm12", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "pshufd xmm4, xmm3, 245", + "pmuludq xmm3, xmm11", + "pmuludq xmm4, xmm11", + "movdqa xmm2, xmm3", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm3, xmm2", + "psrlq xmm3, 32", + "movdqa xmm2, xmm4", + "pmuludq xmm2, xmm14", + "pmuludq xmm2, xmm15", + "paddq xmm4, xmm2", + "por xmm3, xmm4", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "paddd xmm3, xmm5", + "psubd xmm3, xmm15", + "movdqa xmm2, xmm3", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm3, xmm2", + "lfence", + "mov DWORD PTR [r8+1016], r11d", + "ldmxcsr DWORD PTR [r8+1016]", + "movdqu XMMWORD PTR [rdi], xmm3", "ret", ) } @@ -1513,6 +1681,8 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_add_ntt(h: *mut [u32; 256 /// /// Contract: `VG.Spec.MlDsa.addContract`. Constant time: only the pointers may affect timing, not the data. /// +/// The function computes on four coefficients at a time in SSE2 registers. +/// /// # Safety /// /// * `f` must be valid for reads and writes of 1024 bytes. @@ -1524,17 +1694,22 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_add_ntt(h: *mut [u32; 256 #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa_add(f: *mut [u32; 256], g: *const [u32; 256]) { core::arch::naked_asm!( - "mov ecx, 256", + "mov eax, 8380417", + "movq xmm15, rax", + "pshufd xmm15, xmm15, 0", + "mov ecx, 64", "20:", - "mov eax, DWORD PTR [rdi]", - "add eax, DWORD PTR [rsi]", - "sub eax, 8380417", - "sbb edx, edx", - "and edx, 8380417", - "add eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdi]", + "movdqu xmm1, XMMWORD PTR [rsi]", + "paddd xmm0, xmm1", + "psubd xmm0, xmm15", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "movdqu XMMWORD PTR [rdi], xmm0", + "add rdi, 16", + "add rsi, 16", "sub rcx, 1", "jne 20b", "ret", @@ -1545,6 +1720,8 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_add(f: *mut [u32; 256], g: *const /// /// Contract: `VG.Spec.MlDsa.subContract`. Constant time: only the pointers may affect timing, not the data. /// +/// The function computes on four coefficients at a time in SSE2 registers. +/// /// # Safety /// /// * `f` must be valid for reads and writes of 1024 bytes. @@ -1556,18 +1733,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_add(f: *mut [u32; 256], g: *const #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa_sub(f: *mut [u32; 256], g: *const [u32; 256]) { core::arch::naked_asm!( - "mov ecx, 256", + "mov eax, 8380417", + "movq xmm15, rax", + "pshufd xmm15, xmm15, 0", + "mov ecx, 64", "20:", - "mov eax, DWORD PTR [rdi]", - "add eax, 8380417", - "sub eax, DWORD PTR [rsi]", - "sub eax, 8380417", - "sbb edx, edx", - "and edx, 8380417", - "add eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "add rsi, 4", + "movdqu xmm0, XMMWORD PTR [rdi]", + "movdqu xmm1, XMMWORD PTR [rsi]", + "psubd xmm0, xmm1", + "movdqa xmm2, xmm0", + "psrad xmm2, 31", + "pand xmm2, xmm15", + "paddd xmm0, xmm2", + "movdqu XMMWORD PTR [rdi], xmm0", + "add rdi, 16", + "add rsi, 16", "sub rcx, 1", "jne 20b", "ret", From 6703c7a8ed232a70c06c0688c0105c084297e8ac Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 16:13:28 +0000 Subject: [PATCH 3/8] ML-DSA on x86-64: key generation, signing and verification generic over the arithmetic Key generation, signing and verification on x86-64 now call the polynomial arithmetic of an implementation given as a variant of the new interface MlDsaArith (Variants/MlDsaArith/X86_64/). They are registered in Generic/MlDsaArith/X86_64/ and emitted once for each implementation, so a faster arithmetic (e.g. AVX2) reaches them without editing them. The only variant is the existing SSE2 code (Sse2), and the generated code is unchanged. - Impl: Arith.Backend holds the six functions' code and a name suffix. Each Prims gets a sfx field, and the arithmetic calls are named "vg_mldsa_ntt" ++ sfx and so on. primsWith B swaps in B's arithmetic. - Proof: ArithImpl is a backend with FnOk for each function: verified without stack, no rsp writes, depth <= 2, ctlOk, spSafe. prims_okWith builds each caller's PrimsOk from it. - ctlOk and spSafe of signing and key generation are no longer evaluated on the whole code with a concrete backend. Same/same_tac (Proof/MlDsa/X86_64/Arith/Same.lean) shows that a check composing over the code's structure (ctlC, Code.allInstrs) gives the same result as on the code with every arithmetic function empty, which the kernel evaluates once per parameter set. Verification already did this by hand; its call lemmas now allow the names to differ. - The registration files Artifacts/MlDsa{KeyGen,Sign,Verify}/X86_64.lean move to Generic/MlDsaArith/X86_64/. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- .../Artifacts/MlDsaKeyGen/X86_64.lean | 53 ---------- .../Artifacts/MlDsaSign/X86_64.lean | 55 ---------- .../Artifacts/MlDsaVerify/X86_64.lean | 56 ---------- .../MlDsaArith/X86_64/MlDsaKeyGen.lean | 65 ++++++++++++ .../Generic/MlDsaArith/X86_64/MlDsaSign.lean | 65 ++++++++++++ .../MlDsaArith/X86_64/MlDsaVerify.lean | 66 ++++++++++++ .../Impl/MlDsa/X86_64/Arith/Backend.lean | 41 +++++++ .../Impl/MlDsa/X86_64/KeyGen/Inst.lean | 22 ++-- .../Impl/MlDsa/X86_64/KeyGen/KeyGen.lean | 28 ++--- .../Impl/MlDsa/X86_64/KeyGen/Prims.lean | 2 + .../Impl/MlDsa/X86_64/Sign/Frag.lean | 14 +-- .../Impl/MlDsa/X86_64/Verify/Frag.lean | 12 ++- .../Proof/MlDsa/X86_64/Arith/Backend.lean | 74 +++++++++++++ .../Proof/MlDsa/X86_64/Arith/Same.lean | 100 ++++++++++++++++++ .../Proof/MlDsa/X86_64/KeyGen/Call.lean | 24 ++--- .../Proof/MlDsa/X86_64/KeyGen/Inst.lean | 82 +++++++++----- .../Proof/MlDsa/X86_64/KeyGen/RestRow.lean | 16 +-- .../Proof/MlDsa/X86_64/Sign/Inst.lean | 82 +++++++++----- .../Proof/MlDsa/X86_64/Sign/Verified.lean | 84 ++++++++------- .../Proof/MlDsa/X86_64/Verify/Instrs.lean | 53 +++++----- .../Proof/MlDsa/X86_64/Verify/Prims.lean | 95 ++++++++++------- .../Variants/MlDsaArith/X86_64/Sse2.lean | 16 +++ 22 files changed, 730 insertions(+), 375 deletions(-) delete mode 100644 lean/VerifiedGarbage/Artifacts/MlDsaKeyGen/X86_64.lean delete mode 100644 lean/VerifiedGarbage/Artifacts/MlDsaSign/X86_64.lean delete mode 100644 lean/VerifiedGarbage/Artifacts/MlDsaVerify/X86_64.lean create mode 100644 lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaKeyGen.lean create mode 100644 lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaSign.lean create mode 100644 lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean create mode 100644 lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Same.lean create mode 100644 lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Sse2.lean diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaKeyGen/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaKeyGen/X86_64.lean deleted file mode 100644 index 5760b7b53..000000000 --- a/lean/VerifiedGarbage/Artifacts/MlDsaKeyGen/X86_64.lean +++ /dev/null @@ -1,53 +0,0 @@ -import VerifiedGarbage.TCB.X86_64.Target -import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Inst - -/-! -# ML-DSA (FIPS 204) key generation on x86-64 - -A registration file (see `TCB/Emit.lean`): the artifacts it lists are -emitted. **Review note**: `sig` and `doc` are trusted, as they tie the Rust -caller to the contract; check them against the contract's `pre`/`post`. Each -artifact is made from its function's `Api` (in `Spec/MlDsa/Contract.lean`, -reviewed with the contract), and this file adds only notes on the -implementation. The emitter adds the `# Safety` items that depend on the -target (`Sig.layoutDoc`), from `stack` and `writeArgs`, which `ofSig` checks -against the contract. --/ - -namespace VG.Artifacts.MlDsaKeyGen.X86_64 - -/-- Notes on the implementation, the same for every parameter set. -/ -def notes : List String := - ["The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 \ - bytes of stack below its return address.", - "It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and \ - zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not \ - depend on whether key generation fails."] - -def artifacts : List Artifact := [ - { Spec.MlDsa.keyGen44Api with - target := X86_64.target - doc := Spec.MlDsa.keyGen44Api.doc (notes := notes) - code := Impl.MlDsa.X86_64.KeyGen.keyGen44 - contract := Spec.MlDsa.keyGenContract Spec.MlDsa.mlDsa44 X86_64.abi 32 - stack := 32 - verified := Proof.MlDsa.X86_64.KeyGen.keyGen44_verified - spSafe := Code.all_of_allInstrs (by decide +kernel) }, - { Spec.MlDsa.keyGen65Api with - target := X86_64.target - doc := Spec.MlDsa.keyGen65Api.doc (notes := notes) - code := Impl.MlDsa.X86_64.KeyGen.keyGen65 - contract := Spec.MlDsa.keyGenContract Spec.MlDsa.mlDsa65 X86_64.abi 32 - stack := 32 - verified := Proof.MlDsa.X86_64.KeyGen.keyGen65_verified - spSafe := Code.all_of_allInstrs (by decide +kernel) }, - { Spec.MlDsa.keyGen87Api with - target := X86_64.target - doc := Spec.MlDsa.keyGen87Api.doc (notes := notes) - code := Impl.MlDsa.X86_64.KeyGen.keyGen87 - contract := Spec.MlDsa.keyGenContract Spec.MlDsa.mlDsa87 X86_64.abi 32 - stack := 32 - verified := Proof.MlDsa.X86_64.KeyGen.keyGen87_verified - spSafe := Code.all_of_allInstrs (by decide +kernel) }] - -end VG.Artifacts.MlDsaKeyGen.X86_64 diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaSign/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaSign/X86_64.lean deleted file mode 100644 index c64c90d20..000000000 --- a/lean/VerifiedGarbage/Artifacts/MlDsaSign/X86_64.lean +++ /dev/null @@ -1,55 +0,0 @@ -import VerifiedGarbage.TCB.X86_64.Target -import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.Verified - -/-! -# ML-DSA (FIPS 204) signing on x86-64 - -A registration file (see `TCB/Emit.lean`): the artifacts it lists are -emitted. **Review note**: `sig` and `doc` are trusted, as they tie the Rust -caller to the contract; check them against the contract's `pre`/`post`. Each -artifact is made from its function's `Api` (in `Spec/MlDsa/Contract.lean`, -reviewed with the contract), and this file adds only notes on the -implementation. The emitter adds the `# Safety` items that depend on the -target (`Sig.layoutDoc`), from `stack` and `writeArgs`, which `ofSig` checks -against the contract. --/ - -namespace VG.Artifacts.MlDsaSign.X86_64 - -open VG.Proof.MlDsa.X86_64.Sign (prims) - -/-- Notes on the implementation, the same for every parameter set. -/ -def notes : List String := - ["The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 \ - bytes of stack below its return address.", - "The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes \ - every validity check and combines them without branching: the one branch on their result \ - is the only place an iteration's outcome affects timing."] - -def artifacts : List Artifact := [ - { Spec.MlDsa.sign44Api with - target := X86_64.target - doc := Spec.MlDsa.sign44Api.doc (notes := notes) - code := Impl.MlDsa.X86_64.Sign.sign prims Spec.MlDsa.mlDsa44 - contract := Spec.MlDsa.signContract Spec.MlDsa.mlDsa44 X86_64.abi 24 - stack := 24 - verified := Proof.MlDsa.X86_64.Sign.sign44_verified' - spSafe := Proof.MlDsa.X86_64.Sign.sign44_spSafe }, - { Spec.MlDsa.sign65Api with - target := X86_64.target - doc := Spec.MlDsa.sign65Api.doc (notes := notes) - code := Impl.MlDsa.X86_64.Sign.sign prims Spec.MlDsa.mlDsa65 - contract := Spec.MlDsa.signContract Spec.MlDsa.mlDsa65 X86_64.abi 24 - stack := 24 - verified := Proof.MlDsa.X86_64.Sign.sign65_verified' - spSafe := Proof.MlDsa.X86_64.Sign.sign65_spSafe }, - { Spec.MlDsa.sign87Api with - target := X86_64.target - doc := Spec.MlDsa.sign87Api.doc (notes := notes) - code := Impl.MlDsa.X86_64.Sign.sign prims Spec.MlDsa.mlDsa87 - contract := Spec.MlDsa.signContract Spec.MlDsa.mlDsa87 X86_64.abi 24 - stack := 24 - verified := Proof.MlDsa.X86_64.Sign.sign87_verified' - spSafe := Proof.MlDsa.X86_64.Sign.sign87_spSafe }] - -end VG.Artifacts.MlDsaSign.X86_64 diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaVerify/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaVerify/X86_64.lean deleted file mode 100644 index bb1ea07ef..000000000 --- a/lean/VerifiedGarbage/Artifacts/MlDsaVerify/X86_64.lean +++ /dev/null @@ -1,56 +0,0 @@ -import VerifiedGarbage.TCB.X86_64.Target -import VerifiedGarbage.Proof.MlDsa.X86_64.Verify.Prims - -/-! -# ML-DSA (FIPS 204) on x86-64: verification - -A registration file (see `TCB/Emit.lean`): the artifacts it lists are -emitted. **Review note**: `sig` and `doc` are trusted, as they tie the Rust -caller to the contract; check them against the contract's `pre`/`post`. Each -artifact is made from its function's `Api` (in `Spec/MlDsa/Contract.lean`, -reviewed with the contract), and this file adds only notes on the -implementation. The emitter adds the `# Safety` items that depend on the -target (`Sig.layoutDoc`), from `stack` and `writeArgs`, which `ofSig` checks -against the contract. - -The stack is 24 bytes: the return address of a call of a primitive, and up -to 16 bytes for its own calls. --/ - -namespace VG.Artifacts.MlDsaVerify.X86_64 - -open VG -open VG.Proof.MlDsa.X86_64.Verify (prims prims_ok verify_prims verify_spSafe) - -/-- What the documentation says of the implementation. -/ -def note : String := - "It calls the `vg_mldsa_*` primitives and the SHAKE256 sponge. The samplers' results are combined \ - without a branch, so the only branches depend on the public key and the signature." - -def artifacts : List Artifact := [ - { Spec.MlDsa.verify44Api with - target := X86_64.target - doc := Spec.MlDsa.verify44Api.doc (notes := [note]) - code := Impl.MlDsa.X86_64.Verify.verify prims Spec.MlDsa.mlDsa44 - contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa44 X86_64.abi 24 - stack := 24 - verified := verify_prims (List.mem_cons_self ..) - spSafe := verify_spSafe prims_ok (List.mem_cons_self ..) }, - { Spec.MlDsa.verify65Api with - target := X86_64.target - doc := Spec.MlDsa.verify65Api.doc (notes := [note]) - code := Impl.MlDsa.X86_64.Verify.verify prims Spec.MlDsa.mlDsa65 - contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa65 X86_64.abi 24 - stack := 24 - verified := verify_prims (List.mem_cons_of_mem _ (List.mem_cons_self ..)) - spSafe := verify_spSafe prims_ok (List.mem_cons_of_mem _ (List.mem_cons_self ..)) }, - { Spec.MlDsa.verify87Api with - target := X86_64.target - doc := Spec.MlDsa.verify87Api.doc (notes := [note]) - code := Impl.MlDsa.X86_64.Verify.verify prims Spec.MlDsa.mlDsa87 - contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa87 X86_64.abi 24 - stack := 24 - verified := verify_prims (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))) - spSafe := verify_spSafe prims_ok (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))) }] - -end VG.Artifacts.MlDsaVerify.X86_64 diff --git a/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaKeyGen.lean b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaKeyGen.lean new file mode 100644 index 000000000..bddd1b047 --- /dev/null +++ b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaKeyGen.lean @@ -0,0 +1,65 @@ +import VerifiedGarbage.TCB.X86_64.Target +import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Inst + +/-! +# ML-DSA (FIPS 204) on x86-64: key generation + +A generic file (see `TCB/Emit.lean`): the artifacts it lists, which call an +implementation `v` of the polynomial arithmetic (`vg_mldsa_ntt`, …), are +emitted once for each implementation (`Variants/MlDsaArith/X86_64/`), named +with its suffix (e.g. `vg_mldsa44_keygen_avx2`), and need its CPU features. +**Review note**: `sig` and `doc` are trusted, as they tie the Rust caller to +the contract; check them against the contract's `pre`/`post`. Each artifact +is made from its function's `Api` (in `Spec/MlDsa/Contract.lean`, reviewed +with the contract), and this file adds only notes on the implementation. The +emitter adds the `# Safety` items that depend on the target +(`Sig.layoutDoc`), from `stack` and `writeArgs`, which `ofSig` checks +against the contract. +-/ + +namespace VG.Generic.MlDsaArith.X86_64.MlDsaKeyGen + +open VG.Proof.MlDsa.X86_64 (ArithImpl) +open VG.Impl.MlDsa.X86_64.KeyGen (keyGen primsWith) + +/-- Notes on the implementation, the same for every parameter set. -/ +def notes : List String := + ["The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 \ + bytes of stack below its return address.", + "It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and \ + zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not \ + depend on whether key generation fails."] + +def artifacts (v : ArithImpl) : List Artifact := [ + { Spec.MlDsa.keyGen44Api with + name := Spec.MlDsa.keyGen44Api.name ++ v.code.sfx + features := v.features + target := X86_64.target + doc := Spec.MlDsa.keyGen44Api.doc (notes := notes) + code := keyGen (primsWith v.code) Spec.MlDsa.mlDsa44 + contract := Spec.MlDsa.keyGenContract Spec.MlDsa.mlDsa44 X86_64.abi 32 + stack := 32 + verified := Proof.MlDsa.X86_64.KeyGen.keyGen_verifiedWith v (.inl rfl) + spSafe := Proof.MlDsa.X86_64.KeyGen.keyGen_spSafe v (.inl rfl) }, + { Spec.MlDsa.keyGen65Api with + name := Spec.MlDsa.keyGen65Api.name ++ v.code.sfx + features := v.features + target := X86_64.target + doc := Spec.MlDsa.keyGen65Api.doc (notes := notes) + code := keyGen (primsWith v.code) Spec.MlDsa.mlDsa65 + contract := Spec.MlDsa.keyGenContract Spec.MlDsa.mlDsa65 X86_64.abi 32 + stack := 32 + verified := Proof.MlDsa.X86_64.KeyGen.keyGen_verifiedWith v (.inr (.inl rfl)) + spSafe := Proof.MlDsa.X86_64.KeyGen.keyGen_spSafe v (.inr (.inl rfl)) }, + { Spec.MlDsa.keyGen87Api with + name := Spec.MlDsa.keyGen87Api.name ++ v.code.sfx + features := v.features + target := X86_64.target + doc := Spec.MlDsa.keyGen87Api.doc (notes := notes) + code := keyGen (primsWith v.code) Spec.MlDsa.mlDsa87 + contract := Spec.MlDsa.keyGenContract Spec.MlDsa.mlDsa87 X86_64.abi 32 + stack := 32 + verified := Proof.MlDsa.X86_64.KeyGen.keyGen_verifiedWith v (.inr (.inr rfl)) + spSafe := Proof.MlDsa.X86_64.KeyGen.keyGen_spSafe v (.inr (.inr rfl)) }] + +end VG.Generic.MlDsaArith.X86_64.MlDsaKeyGen diff --git a/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaSign.lean b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaSign.lean new file mode 100644 index 000000000..2a231e969 --- /dev/null +++ b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaSign.lean @@ -0,0 +1,65 @@ +import VerifiedGarbage.TCB.X86_64.Target +import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.Verified + +/-! +# ML-DSA (FIPS 204) on x86-64: signing + +A generic file (see `TCB/Emit.lean`): the artifacts it lists, which call an +implementation `v` of the polynomial arithmetic (`vg_mldsa_ntt`, …), are +emitted once for each implementation (`Variants/MlDsaArith/X86_64/`), named +with its suffix (e.g. `vg_mldsa44_sign_avx2`), and need its CPU features. +**Review note**: `sig` and `doc` are trusted, as they tie the Rust caller to +the contract; check them against the contract's `pre`/`post`. Each artifact +is made from its function's `Api` (in `Spec/MlDsa/Contract.lean`, reviewed +with the contract), and this file adds only notes on the implementation. The +emitter adds the `# Safety` items that depend on the target +(`Sig.layoutDoc`), from `stack` and `writeArgs`, which `ofSig` checks +against the contract. +-/ + +namespace VG.Generic.MlDsaArith.X86_64.MlDsaSign + +open VG.Proof.MlDsa.X86_64 (ArithImpl) +open VG.Proof.MlDsa.X86_64.Sign (primsWith) + +/-- Notes on the implementation, the same for every parameter set. -/ +def notes : List String := + ["The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 \ + bytes of stack below its return address.", + "The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes \ + every validity check and combines them without branching: the one branch on their result \ + is the only place an iteration's outcome affects timing."] + +def artifacts (v : ArithImpl) : List Artifact := [ + { Spec.MlDsa.sign44Api with + name := Spec.MlDsa.sign44Api.name ++ v.code.sfx + features := v.features + target := X86_64.target + doc := Spec.MlDsa.sign44Api.doc (notes := notes) + code := Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) Spec.MlDsa.mlDsa44 + contract := Spec.MlDsa.signContract Spec.MlDsa.mlDsa44 X86_64.abi 24 + stack := 24 + verified := Proof.MlDsa.X86_64.Sign.sign_verified' v (.inl rfl) + spSafe := Proof.MlDsa.X86_64.Sign.sign_spSafe v (.inl rfl) }, + { Spec.MlDsa.sign65Api with + name := Spec.MlDsa.sign65Api.name ++ v.code.sfx + features := v.features + target := X86_64.target + doc := Spec.MlDsa.sign65Api.doc (notes := notes) + code := Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) Spec.MlDsa.mlDsa65 + contract := Spec.MlDsa.signContract Spec.MlDsa.mlDsa65 X86_64.abi 24 + stack := 24 + verified := Proof.MlDsa.X86_64.Sign.sign_verified' v (.inr (.inl rfl)) + spSafe := Proof.MlDsa.X86_64.Sign.sign_spSafe v (.inr (.inl rfl)) }, + { Spec.MlDsa.sign87Api with + name := Spec.MlDsa.sign87Api.name ++ v.code.sfx + features := v.features + target := X86_64.target + doc := Spec.MlDsa.sign87Api.doc (notes := notes) + code := Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) Spec.MlDsa.mlDsa87 + contract := Spec.MlDsa.signContract Spec.MlDsa.mlDsa87 X86_64.abi 24 + stack := 24 + verified := Proof.MlDsa.X86_64.Sign.sign_verified' v (.inr (.inr rfl)) + spSafe := Proof.MlDsa.X86_64.Sign.sign_spSafe v (.inr (.inr rfl)) }] + +end VG.Generic.MlDsaArith.X86_64.MlDsaSign diff --git a/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean new file mode 100644 index 000000000..903d1f596 --- /dev/null +++ b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean @@ -0,0 +1,66 @@ +import VerifiedGarbage.TCB.X86_64.Target +import VerifiedGarbage.Proof.MlDsa.X86_64.Verify.Prims + +/-! +# ML-DSA (FIPS 204) on x86-64: verification + +A generic file (see `TCB/Emit.lean`): the artifacts it lists, which call an +implementation `v` of the polynomial arithmetic (`vg_mldsa_ntt`, …), are +emitted once for each implementation (`Variants/MlDsaArith/X86_64/`), named +with its suffix (e.g. `vg_mldsa44_verify_avx2`), and need its CPU features. +**Review note**: `sig` and `doc` are trusted, as they tie the Rust caller to +the contract; check them against the contract's `pre`/`post`. Each artifact +is made from its function's `Api` (in `Spec/MlDsa/Contract.lean`, reviewed +with the contract), and this file adds only notes on the implementation. The +emitter adds the `# Safety` items that depend on the target +(`Sig.layoutDoc`), from `stack` and `writeArgs`, which `ofSig` checks +against the contract. + +The stack is 24 bytes: the return address of a call of a primitive, and up +to 16 bytes for its own calls. +-/ + +namespace VG.Generic.MlDsaArith.X86_64.MlDsaVerify + +open VG +open VG.Proof.MlDsa.X86_64 (ArithImpl) +open VG.Proof.MlDsa.X86_64.Verify (primsWith prims_okWith verify_prims verify_spSafe) + +/-- What the documentation says of the implementation. -/ +def note : String := + "It calls the `vg_mldsa_*` primitives and the SHAKE256 sponge. The samplers' results are combined \ + without a branch, so the only branches depend on the public key and the signature." + +def artifacts (v : ArithImpl) : List Artifact := [ + { Spec.MlDsa.verify44Api with + name := Spec.MlDsa.verify44Api.name ++ v.code.sfx + features := v.features + target := X86_64.target + doc := Spec.MlDsa.verify44Api.doc (notes := [note]) + code := Impl.MlDsa.X86_64.Verify.verify (primsWith v.code) Spec.MlDsa.mlDsa44 + contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa44 X86_64.abi 24 + stack := 24 + verified := verify_prims v (List.mem_cons_self ..) + spSafe := verify_spSafe (prims_okWith v) (List.mem_cons_self ..) }, + { Spec.MlDsa.verify65Api with + name := Spec.MlDsa.verify65Api.name ++ v.code.sfx + features := v.features + target := X86_64.target + doc := Spec.MlDsa.verify65Api.doc (notes := [note]) + code := Impl.MlDsa.X86_64.Verify.verify (primsWith v.code) Spec.MlDsa.mlDsa65 + contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa65 X86_64.abi 24 + stack := 24 + verified := verify_prims v (List.mem_cons_of_mem _ (List.mem_cons_self ..)) + spSafe := verify_spSafe (prims_okWith v) (List.mem_cons_of_mem _ (List.mem_cons_self ..)) }, + { Spec.MlDsa.verify87Api with + name := Spec.MlDsa.verify87Api.name ++ v.code.sfx + features := v.features + target := X86_64.target + doc := Spec.MlDsa.verify87Api.doc (notes := [note]) + code := Impl.MlDsa.X86_64.Verify.verify (primsWith v.code) Spec.MlDsa.mlDsa87 + contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa87 X86_64.abi 24 + stack := 24 + verified := verify_prims v (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))) + spSafe := verify_spSafe (prims_okWith v) (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))) }] + +end VG.Generic.MlDsaArith.X86_64.MlDsaVerify diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean new file mode 100644 index 000000000..c6ef074df --- /dev/null +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean @@ -0,0 +1,41 @@ +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Ntt +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Mul +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.AddSub + +/-! +# ML-DSA on x86-64: implementations of the polynomial arithmetic + +Key generation, signing and verification call the polynomial arithmetic of +one implementation, a `Backend`: the code of `vg_mldsa_ntt`, +`vg_mldsa_inv_ntt`, `vg_mldsa_multiply_ntt`, `vg_mldsa_multiply_add_ntt`, +`vg_mldsa_add` and `vg_mldsa_sub`, whose names end with `sfx` (e.g. +`_avx2`; nothing for the SSE2 code, `sse2`). Each is a variant of the +interface `MlDsaArith` on x86-64 (`Variants/MlDsaArith/X86_64/`), and the +functions that call them are emitted once for each +(`Generic/MlDsaArith/X86_64/`). +-/ + +namespace VG.Impl.MlDsa.X86_64.Arith + +open VG.X86_64 + +/-- An implementation of the polynomial arithmetic. -/ +structure Backend where + ntt : Prog isa + invNtt : Prog isa + mul : Prog isa + mulAdd : Prog isa + add : Prog isa + sub : Prog isa + /-- What the names of its functions, and of those calling them, end with. -/ + sfx : String + +/-- The SSE2 code. -/ +def Backend.sse2 : Backend := ⟨Arith.ntt, Arith.nttInv, Arith.mul, Arith.mulAdd, Arith.add, Arith.sub, ""⟩ + +/-- Every function empty, which the proofs that the functions calling a +backend never write `rsp` (and load MXCSR only to restore it) evaluate in +its place. -/ +def Backend.empty : Backend := ⟨.block [], .block [], .block [], .block [], .block [], .block [], ""⟩ + +end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean index f4a1d0c87..5206623ad 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Inst.lean @@ -1,7 +1,5 @@ import VerifiedGarbage.Impl.MlDsa.X86_64.KeyGen.KeyGen -import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Ntt -import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Mul -import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.AddSub +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Backend import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejNtt import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejBounded import VerifiedGarbage.Impl.MlDsa.X86_64.Round.Round @@ -11,7 +9,8 @@ import VerifiedGarbage.Impl.MlDsa.X86_64.Pack.Encode # ML-DSA key generation on x86-64, with this library's primitives `keyGen` (`KeyGen.lean`) called with the x86-64 implementations of the -primitives it calls (`Arith/`, `Sample/`, `Round/`, `Pack/`). +primitives it calls (`Arith/`, `Sample/`, `Round/`, `Pack/`), with the +polynomial arithmetic of a `Backend` (`primsWith`). -/ namespace VG.Impl.MlDsa.X86_64.KeyGen @@ -31,11 +30,14 @@ def prims : Prims where simpleBitPack := Pack.simpleBitPack bitPack := Pack.bitPack -/-- `vg_mldsa44_keygen` -/ -def keyGen44 : Prog isa := keyGen prims Spec.MlDsa.mlDsa44 -/-- `vg_mldsa65_keygen` -/ -def keyGen65 : Prog isa := keyGen prims Spec.MlDsa.mlDsa65 -/-- `vg_mldsa87_keygen` -/ -def keyGen87 : Prog isa := keyGen prims Spec.MlDsa.mlDsa87 +/-- The primitives, with the polynomial arithmetic of `B`. -/ +def primsWith (B : Arith.Backend) : Prims := + { prims with + ntt := B.ntt + invNtt := B.invNtt + mul := B.mul + mulAdd := B.mulAdd + add := B.add + sfx := B.sfx } end VG.Impl.MlDsa.X86_64.KeyGen diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean index 06126c30f..4f3d0ef4c 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/KeyGen.lean @@ -71,20 +71,20 @@ def oT0 (p : Params) : Nat := 128 + lenS p * (p.ℓ + p.k) /-- `r ← v`, a 32-bit immediate. -/ def imm (r : Reg) (v : Nat) : List Instr := [.mov32 r (.imm (BitVec.ofNat 32 v))] -def nttAt (c : Prog isa) (f : Ptr) : Prog isa := - .seq (.block (lea .rdi f ++ lea .rsi (sc oSS))) (.call "vg_mldsa_ntt" c) +def nttAt (sfx : String) (c : Prog isa) (f : Ptr) : Prog isa := + .seq (.block (lea .rdi f ++ lea .rsi (sc oSS))) (.call ("vg_mldsa_ntt" ++ sfx) c) -def invNttAt (c : Prog isa) (f : Ptr) : Prog isa := - .seq (.block (lea .rdi f ++ lea .rsi (sc oSS))) (.call "vg_mldsa_inv_ntt" c) +def invNttAt (sfx : String) (c : Prog isa) (f : Ptr) : Prog isa := + .seq (.block (lea .rdi f ++ lea .rsi (sc oSS))) (.call ("vg_mldsa_inv_ntt" ++ sfx) c) -def mulAt (c : Prog isa) (h f g : Ptr) : Prog isa := - .seq (.block (lea .rdi h ++ lea .rsi f ++ lea .rdx g)) (.call "vg_mldsa_multiply_ntt" c) +def mulAt (sfx : String) (c : Prog isa) (h f g : Ptr) : Prog isa := + .seq (.block (lea .rdi h ++ lea .rsi f ++ lea .rdx g)) (.call ("vg_mldsa_multiply_ntt" ++ sfx) c) -def mulAddAt (c : Prog isa) (h f g : Ptr) : Prog isa := - .seq (.block (lea .rdi h ++ lea .rsi f ++ lea .rdx g)) (.call "vg_mldsa_multiply_add_ntt" c) +def mulAddAt (sfx : String) (c : Prog isa) (h f g : Ptr) : Prog isa := + .seq (.block (lea .rdi h ++ lea .rsi f ++ lea .rdx g)) (.call ("vg_mldsa_multiply_add_ntt" ++ sfx) c) -def addAt (c : Prog isa) (f g : Ptr) : Prog isa := - .seq (.block (lea .rdi f ++ lea .rsi g)) (.call "vg_mldsa_add" c) +def addAt (sfx : String) (c : Prog isa) (f g : Ptr) : Prog isa := + .seq (.block (lea .rdi f ++ lea .rsi g)) (.call ("vg_mldsa_add" ++ sfx) c) def rejNttAt (c : Prog isa) (seed a : Ptr) : Prog isa := .seq (.block (lea .rdi seed ++ lea .rsi a ++ lea .rdx (sc oSS))) (.call "vg_mldsa_rej_ntt_poly" c) @@ -141,13 +141,13 @@ def packS (P : Prims) (p : Params) (r : Nat) : Prog isa := bitPackAt P.bitPack (sP p r) p.η p.η (.r13, 128 + lenS p * r) (lenS p) /-- `ŝ₁[j] = NTT(s₁[j])`. -/ -def nttS (P : Prims) (p : Params) (j : Nat) : Prog isa := nttAt P.ntt (sP p j) +def nttS (P : Prims) (p : Params) (j : Nat) : Prog isa := nttAt P.sfx P.ntt (sP p j) /-- Row `i`: `t = NTT⁻¹(Σⱼ Â[i, j] ŝ₁[j]) + s₂[i]`, and its `t₁` to `pk` and `t₀` to `sk`. -/ def row (P : Prims) (p : Params) (i : Nat) : Prog isa := - .seq (mulAt P.mul (tP p) (aP (p.ℓ * i)) (sP p 0)) - (.seq (seqR (fun j => mulAddAt P.mulAdd (tP p) (aP (p.ℓ * i + j)) (sP p j)) 1 (p.ℓ - 1)) - (.seq (invNttAt P.invNtt (tP p)) (.seq (addAt P.add (tP p) (sP p (p.ℓ + i))) + .seq (mulAt P.sfx P.mul (tP p) (aP (p.ℓ * i)) (sP p 0)) + (.seq (seqR (fun j => mulAddAt P.sfx P.mulAdd (tP p) (aP (p.ℓ * i + j)) (sP p j)) 1 (p.ℓ - 1)) + (.seq (invNttAt P.sfx P.invNtt (tP p)) (.seq (addAt P.sfx P.add (tP p) (sP p (p.ℓ + i))) (.seq (power2RoundAt P.power2Round (tP p) (t1P p) (t0P p)) (.seq (simpleBitPackAt P.simpleBitPack (t1P p) 1023 (.r12, 32 + 320 * i) 320) (bitPackAt P.bitPack (t0P p) 4095 4096 (.r13, oT0 p + 416 * i) 416)))))) diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean index a3e13476d..a2ed604be 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/KeyGen/Prims.lean @@ -36,5 +36,7 @@ structure Prims where simpleBitPack : Prog isa /-- `vg_mldsa_bit_pack` -/ bitPack : Prog isa + /-- What the names of the polynomial arithmetic's functions end with (`Arith.Backend`). -/ + sfx : String := "" end VG.Impl.MlDsa.X86_64.KeyGen diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean index 11dc35463..4fa71ab43 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean @@ -44,6 +44,8 @@ structure Prims where bitPack : Prog isa bitUnpack : Prog isa hintBitPack : Prog isa + /-- What the names of the polynomial arithmetic's functions end with (`Arith.Backend`). -/ + sfx : String := "" /-! ## The layout of the working space (in bytes) @@ -165,17 +167,17 @@ Each takes its working space (if any) at `PS`. -/ section variable (P : Prims) -def nttAt (f : Ptr) : Prog isa := callP "vg_mldsa_ntt" P.ntt [.ptr f, .ptr (sc oPS)] +def nttAt (f : Ptr) : Prog isa := callP ("vg_mldsa_ntt" ++ P.sfx) P.ntt [.ptr f, .ptr (sc oPS)] -def invNttAt (f : Ptr) : Prog isa := callP "vg_mldsa_inv_ntt" P.invNtt [.ptr f, .ptr (sc oPS)] +def invNttAt (f : Ptr) : Prog isa := callP ("vg_mldsa_inv_ntt" ++ P.sfx) P.invNtt [.ptr f, .ptr (sc oPS)] -def mulAt (h f g : Ptr) : Prog isa := callP "vg_mldsa_multiply_ntt" P.mul [.ptr h, .ptr f, .ptr g] +def mulAt (h f g : Ptr) : Prog isa := callP ("vg_mldsa_multiply_ntt" ++ P.sfx) P.mul [.ptr h, .ptr f, .ptr g] -def mulAddAt (h f g : Ptr) : Prog isa := callP "vg_mldsa_multiply_add_ntt" P.mulAdd [.ptr h, .ptr f, .ptr g] +def mulAddAt (h f g : Ptr) : Prog isa := callP ("vg_mldsa_multiply_add_ntt" ++ P.sfx) P.mulAdd [.ptr h, .ptr f, .ptr g] -def addAt (f g : Ptr) : Prog isa := callP "vg_mldsa_add" P.add [.ptr f, .ptr g] +def addAt (f g : Ptr) : Prog isa := callP ("vg_mldsa_add" ++ P.sfx) P.add [.ptr f, .ptr g] -def subAt (f g : Ptr) : Prog isa := callP "vg_mldsa_sub" P.sub [.ptr f, .ptr g] +def subAt (f g : Ptr) : Prog isa := callP ("vg_mldsa_sub" ++ P.sfx) P.sub [.ptr f, .ptr g] /-- `RejNTTPoly` of the seed at `RS` to `a`, and `r15 ← r15 ∧ result`. -/ def rejAt (a : Ptr) : Prog isa := diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean index cdeaf14d1..5b4e5e24f 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean @@ -137,22 +137,24 @@ structure Prims where unpackT1 : Prog isa hintUnpack : Prog isa normLt : Prog isa + /-- What the names of the polynomial arithmetic's functions end with (`Arith.Backend`). -/ + sfx : String := "" section variable (P : Prims) -def nttAt (f : Ptr) : Prog isa := callAt "vg_mldsa_ntt" P.ntt [(.rdi, .ptr f), (.rsi, .ptr (sc oSS))] +def nttAt (f : Ptr) : Prog isa := callAt ("vg_mldsa_ntt" ++ P.sfx) P.ntt [(.rdi, .ptr f), (.rsi, .ptr (sc oSS))] def invNttAt (f : Ptr) : Prog isa := - callAt "vg_mldsa_inv_ntt" P.invNtt [(.rdi, .ptr f), (.rsi, .ptr (sc oSS))] + callAt ("vg_mldsa_inv_ntt" ++ P.sfx) P.invNtt [(.rdi, .ptr f), (.rsi, .ptr (sc oSS))] def mulAt (h f g : Ptr) : Prog isa := - callAt "vg_mldsa_multiply_ntt" P.mul [(.rdi, .ptr h), (.rsi, .ptr f), (.rdx, .ptr g)] + callAt ("vg_mldsa_multiply_ntt" ++ P.sfx) P.mul [(.rdi, .ptr h), (.rsi, .ptr f), (.rdx, .ptr g)] def mulAddAt (h f g : Ptr) : Prog isa := - callAt "vg_mldsa_multiply_add_ntt" P.mulAdd [(.rdi, .ptr h), (.rsi, .ptr f), (.rdx, .ptr g)] + callAt ("vg_mldsa_multiply_add_ntt" ++ P.sfx) P.mulAdd [(.rdi, .ptr h), (.rsi, .ptr f), (.rdx, .ptr g)] -def subAt (f g : Ptr) : Prog isa := callAt "vg_mldsa_sub" P.sub [(.rdi, .ptr f), (.rsi, .ptr g)] +def subAt (f g : Ptr) : Prog isa := callAt ("vg_mldsa_sub" ++ P.sfx) P.sub [(.rdi, .ptr f), (.rsi, .ptr g)] def rejNttAt (a : Ptr) : Prog isa := callAt "vg_mldsa_rej_ntt_poly" P.rejNtt [(.rdi, .ptr (sc oSB)), (.rsi, .ptr a), (.rdx, .ptr (sc oSS))] diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean new file mode 100644 index 000000000..6a27d6c37 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean @@ -0,0 +1,74 @@ +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Backend +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Ntt +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub +import VerifiedGarbage.Proof.MlKem.X86_64.ArithOk + +/-! +# ML-DSA on x86-64: what the callers of the polynomial arithmetic need of it + +Untrusted: everything here is checked by Lean. An `ArithImpl` is an +implementation of the polynomial arithmetic (`Impl.MlDsa.X86_64.Arith.Backend`) +with what key generation, signing and verification need of each of its +functions (`FnOk`): it meets its contract without using the stack, never +writes `rsp`, calls no deeper than twice, and loads MXCSR only to restore +it. Each is a variant of the interface `MlDsaArith` on x86-64 +(`Variants/MlDsaArith/X86_64/`), and the functions that call it are proven +once for all of them (`Generic/MlDsaArith/X86_64/`). +-/ + +namespace VG.Proof.MlDsa.X86_64 + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith + +/-- What a caller needs of a function with the contract `k` (given the +stack its calls use) and the code `c`. -/ +structure FnOk (k : Nat → Contract isa) (c : Prog isa) : Prop where + ver : Verified X86_64.target c (k 0) + nosp : NoSp c + depth : c.depth ≤ 2 + ctl : ctlOk c = true + sp : c.all (fun i => !isa.writesSp i) = true + +/-- Each function of the backend `B` meets its contract, and is safe to call. -/ +structure BackendOk (B : Backend) : Prop where + ntt : FnOk (fun S => Spec.MlDsa.nttContract X86_64.abi S) B.ntt + invNtt : FnOk (fun S => Spec.MlDsa.nttInvContract X86_64.abi S) B.invNtt + mul : FnOk (fun S => Spec.MlDsa.mulContract X86_64.abi S) B.mul + mulAdd : FnOk (fun S => Spec.MlDsa.mulAddContract X86_64.abi S) B.mulAdd + add : FnOk (fun S => Spec.MlDsa.addContract X86_64.abi S) B.add + sub : FnOk (fun S => Spec.MlDsa.subContract X86_64.abi S) B.sub + +/-- An implementation of the polynomial arithmetic on x86-64. -/ +structure ArithImpl where + code : Backend + ok : BackendOk code + /-- The CPU features its code requires, which its callers require too. -/ + features : List String + +/-- `FnOk` of code verified without stack, from evaluating it. -/ +theorem FnOk.of {k : Nat → Contract isa} {c : Prog isa} (h : Verified X86_64.target c (k 0)) + (hn : c.allInstrs (fun i => !Taint.clobbers i .rsp) = true) (hd : c.depth ≤ 2) (hc : ctlOk c = true) + (hs : c.allInstrs (fun i => !isa.writesSp i) = true) : FnOk k c := + ⟨h, Proof.MlKem.X86_64.nosp_of hn, hd, hc, Code.all_of_allInstrs hs⟩ + +/-- The SSE2 code. -/ +def ArithImpl.sse2 : ArithImpl where + code := .sse2 + ok := + { ntt := FnOk.of Arith.ntt_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + invNtt := FnOk.of Arith.nttInv_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + mul := FnOk.of Arith.mul_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + mulAdd := FnOk.of Arith.mulAdd_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + add := FnOk.of Arith.add_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + sub := FnOk.of Arith.sub_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) } + features := [] + +end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Same.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Same.lean new file mode 100644 index 000000000..b08c4c356 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Same.lean @@ -0,0 +1,100 @@ +import VerifiedGarbage.Proof.Framework.X86_64.Mxcsr +import VerifiedGarbage.Impl.MlDsa.X86_64.Sign.Frag +import VerifiedGarbage.Impl.MlDsa.X86_64.Verify.Frag +import VerifiedGarbage.Impl.MlKem.X86_64.Frag + +/-! +# ML-DSA on x86-64: checks of code, but for the functions it calls + +Untrusted: everything here is checked by Lean. A check of code that +composes over its structure and looks at the code of each function called +only through `mc` (`Comp m mc`: `ctlC`, with `ctlOk` of the functions +called, and `Code.allInstrs q`) gives the same result on two programs that +differ only in functions called, if it holds (`mc`) of those of the first +and the second calls empty code instead (`Same`, proven by `same_tac` from +the structure of the code). The top-level functions, written for any +implementation of the polynomial arithmetic, are checked once with every +function of it empty, so that the kernel evaluates no implementation of it. +-/ + +namespace VG.Proof.MlDsa.X86_64 + +open VG VG.X86_64 + +/-- `m` composes over the structure of code, and looks at the code of a +function called only through `mc`. -/ +structure Comp (m mc : Prog isa → Bool) : Prop where + seq : ∀ a b, m (.seq a b) = (m a && m b) + ite : ∀ c t e, m (.ite c t e) = (m t && m e) + loop : ∀ b c, m (.loop b c) = m b + call : ∀ n b, m (.call n b) = mc b + nil : mc (.block []) = true + +theorem Comp.ctlC : Comp ctlC ctlOk := + ⟨fun _ _ => rfl, fun _ _ _ => rfl, fun _ _ => rfl, fun _ _ => rfl, rfl⟩ + +theorem Comp.all (q : Instr → Bool) : Comp (Code.allInstrs q) (Code.allInstrs q) := + ⟨fun _ _ => rfl, fun _ _ _ => rfl, fun _ _ => rfl, fun _ _ => rfl, rfl⟩ + +theorem Code.allInstrs_of_all {I C : Type} {q : I → Bool} {c : Code I C} (h : c.all q = true) : + c.allInstrs q = true := by + induction c with + | block is => induction is <;> simp_all [Code.all, Code.allInstrs] + | _ => simp_all [Code.all, Code.allInstrs] + +/-- `m` gives the same result on `c` and `c'`. -/ +def Same (m : Prog isa → Bool) (c c' : Prog isa) : Prop := m c = m c' + +section +variable {m mc : Prog isa → Bool} (hm : Comp m mc) +include hm + +theorem Same.seq {a a' b b' : Prog isa} (ha : Same m a a') (hb : Same m b b') : Same m (.seq a b) (.seq a' b') := by + unfold Same at *; rw [hm.seq, hm.seq, ha, hb] + +theorem Same.ite {c : isa.Cond} {t t' e e' : Prog isa} (ht : Same m t t') (he : Same m e e') : + Same m (.ite c t e) (.ite c t' e') := by + unfold Same at *; rw [hm.ite, hm.ite, ht, he] + +theorem Same.loop {b b' : Prog isa} {c : isa.Cond} (hb : Same m b b') : Same m (.loop b c) (.loop b' c) := by + unfold Same at *; rw [hm.loop, hm.loop, hb] + +theorem Same.call {c : Prog isa} (hc : mc c = true) (n n' : String) : Same m (.call n c) (.call n' (.block [])) := by + unfold Same; rw [hm.call, hm.call, hc, hm.nil] + +theorem Same.seqRS {f g : Nat → Prog isa} (h : ∀ k, Same m (f k) (g k)) : + ∀ a n, Same m (Impl.MlDsa.X86_64.Sign.seqR f a n) (Impl.MlDsa.X86_64.Sign.seqR g a n) + | _, 0 => rfl + | a, n + 1 => Same.seq hm (h a) (Same.seqRS h (a + 1) n) + +theorem Same.seqRV {f g : Nat → Prog isa} (h : ∀ k, Same m (f k) (g k)) : + ∀ a n, Same m (Impl.MlDsa.X86_64.Verify.seqR f a n) (Impl.MlDsa.X86_64.Verify.seqR g a n) + | _, 0 => rfl + | a, n + 1 => Same.seq hm (h a) (Same.seqRV h (a + 1) n) + +theorem Same.seqRK {f g : Nat → Prog isa} (h : ∀ k, Same m (f k) (g k)) : + ∀ a n, Same m (Impl.MlKem.X86_64.seqR f a n) (Impl.MlKem.X86_64.seqR g a n) + | _, 0 => rfl + | a, n + 1 => Same.seq hm (h a) (Same.seqRK h (a + 1) n) + +end + +/-- `m` of `a`, from that of code `b` that it gives the same result on. -/ +theorem Same.ok {m : Prog isa → Bool} {a b : Prog isa} (h : Same m a b) (hb : m b = true) : m a = true := + Eq.trans h hb + +/-- `Same m c c'` for code `c` that calls functions whose `mc` the +hypotheses state, and the same code `c'` but for empty functions in their +place: from the structure of the code. -/ +macro "same_tac " hm:term : tactic => + `(tactic| repeat' (first + | (apply Same.call $hm; assumption) + | apply Same.seq $hm + | apply Same.ite $hm + | apply Same.loop $hm + | (apply Same.seqRS $hm; intro) + | (apply Same.seqRV $hm; intro) + | (apply Same.seqRK $hm; intro) + | rfl)) + +end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean index dda7b3182..f8a03397c 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Call.lean @@ -297,9 +297,9 @@ theorem covers_rw {s : State} {p : Params} (L : Lay kgR (kgW p) s) {a b : Ptr} { covers_append (covers_cons (L.cR ha) covers_nil) (covers_cons (L.cR (inB_mono hb)) covers_nil) include hf hg h1 w1 in -theorem addAt_ok {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.addContract X86_64.abi stk) {s : State} +theorem addAt_ok {sfx : String} {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.addContract X86_64.abi stk) {s : State} (S : Site p s) (rf : Spec.MlDsa.Reduced s.mem (pa s f)) (rg : Spec.MlDsa.Reduced s.mem (pa s g)) : - WP isa (addAt c f g) s fun s' => Post s s' [⟨pa s f, 1024⟩] ∧ MX s' = MX s ∧ + WP isa (addAt sfx c f g) s fun s' => Post s s' [⟨pa s f, 1024⟩] ∧ MX s' = MX s ∧ Spec.MlDsa.PolyIs s'.mem (pa s f) (Spec.MlDsa.add (Spec.MlDsa.polyAt s.mem (pa s f)) (Spec.MlDsa.polyAt s.mem (pa s g))) := by obtain ⟨i1, i2, _⟩ := sepB_spec h1 @@ -314,10 +314,10 @@ theorem addAt_ok {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.addContract rwa [ce_polyAt (by rw [hsp]; exact L.stkD i1), ce_polyAt (by rw [hsp]; exact L.stkD i2), hm] at hpost include hf hg h1 w1 in -theorem addAt_tr {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.addContract X86_64.abi stk) +theorem addAt_tr {sfx : String} {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.addContract X86_64.abi stk) (hbf : f.1 ∈ kgRegs) (hbg : g.1 ∈ kgRegs) : RelCT isa (fun x y => Two p x y ∧ (Spec.MlDsa.Reduced x.mem (pa x f) ∧ Spec.MlDsa.Reduced x.mem (pa x g)) ∧ - (Spec.MlDsa.Reduced y.mem (pa y f) ∧ Spec.MlDsa.Reduced y.mem (pa y g))) (addAt c f g) fun _ _ => True := by + (Spec.MlDsa.Reduced y.mem (pa y f) ∧ Spec.MlDsa.Reduced y.mem (pa y g))) (addAt sfx c f g) fun _ _ => True := by obtain ⟨i1, i2, _⟩ := sepB_spec h1 refine primTr hc (nomem_append (lea_nomem _ _) (lea_nomem _ _)) (fun x y _ => ⟨glue2_ok hf hg x, glue2_ok hf hg y⟩) fun stk hs x y x1 y1 ⟨T, rx, ry⟩ ⟨⟨hv1, hm1⟩, k1⟩ ⟨⟨hv2, hm2⟩, k2⟩ => @@ -362,9 +362,9 @@ theorem mul_pre {stk : Nat} (hstk : stk ≤ 16) {s s1 : State} (S : Site p s) · exact (ce_reduced (by rw [hsp]; exact L.stkD i3)).mpr (hm ▸ rg) include hh hf hg h1 h2 w1 in -theorem mulAt_ok {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulContract X86_64.abi stk) {s : State} +theorem mulAt_ok {sfx : String} {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulContract X86_64.abi stk) {s : State} (S : Site p s) (rf : Spec.MlDsa.Reduced s.mem (pa s f)) (rg : Spec.MlDsa.Reduced s.mem (pa s g)) : - WP isa (mulAt c h f g) s fun s' => Post s s' [⟨pa s h, 1024⟩] ∧ MX s' = MX s ∧ + WP isa (mulAt sfx c h f g) s fun s' => Post s s' [⟨pa s h, 1024⟩] ∧ MX s' = MX s ∧ Spec.MlDsa.PolyIs s'.mem (pa s h) (Spec.MlDsa.multiplyNTT (Spec.MlDsa.polyAt s.mem (pa s f)) (Spec.MlDsa.polyAt s.mem (pa s g))) := by obtain ⟨i1, i2, _⟩ := sepB_spec h1 obtain ⟨_, i3, _⟩ := sepB_spec h2 @@ -379,9 +379,9 @@ theorem mulAt_ok {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulContract rwa [ce_polyAt (by rw [hsp]; exact L.stkD i2), ce_polyAt (by rw [hsp]; exact L.stkD i3), hm] at hpost include hh hf hg h1 h2 w1 in -theorem mulAt_tr {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulContract X86_64.abi stk) +theorem mulAt_tr {sfx : String} {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulContract X86_64.abi stk) (hbh : h.1 ∈ kgRegs) (hbf : f.1 ∈ kgRegs) (hbg : g.1 ∈ kgRegs) : - RelCT isa (fun x y => Two p x y ∧ (Spec.MlDsa.Reduced x.mem (pa x f) ∧ Spec.MlDsa.Reduced x.mem (pa x g)) ∧ (Spec.MlDsa.Reduced y.mem (pa y f) ∧ Spec.MlDsa.Reduced y.mem (pa y g))) (mulAt c h f g) fun _ _ => True := by + RelCT isa (fun x y => Two p x y ∧ (Spec.MlDsa.Reduced x.mem (pa x f) ∧ Spec.MlDsa.Reduced x.mem (pa x g)) ∧ (Spec.MlDsa.Reduced y.mem (pa y f) ∧ Spec.MlDsa.Reduced y.mem (pa y g))) (mulAt sfx c h f g) fun _ _ => True := by obtain ⟨i1, i2, _⟩ := sepB_spec h1 obtain ⟨_, i3, _⟩ := sepB_spec h2 refine primTr hc (nomem_append (nomem_append (lea_nomem _ _) (lea_nomem _ _)) (lea_nomem _ _)) @@ -425,9 +425,9 @@ theorem mulAdd_pre {stk : Nat} (hstk : stk ≤ 16) {s s1 : State} (S : Site p s) · exact (ce_reduced (by rw [hsp]; exact L.stkD i3)).mpr (hm ▸ rg) include hh hf hg h1 h2 w1 in -theorem mulAddAt_ok {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulAddContract X86_64.abi stk) {s : State} +theorem mulAddAt_ok {sfx : String} {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulAddContract X86_64.abi stk) {s : State} (S : Site p s) (rh : Spec.MlDsa.Reduced s.mem (pa s h)) (rf : Spec.MlDsa.Reduced s.mem (pa s f)) (rg : Spec.MlDsa.Reduced s.mem (pa s g)) : - WP isa (mulAddAt c h f g) s fun s' => Post s s' [⟨pa s h, 1024⟩] ∧ MX s' = MX s ∧ + WP isa (mulAddAt sfx c h f g) s fun s' => Post s s' [⟨pa s h, 1024⟩] ∧ MX s' = MX s ∧ Spec.MlDsa.PolyIs s'.mem (pa s h) (Spec.MlDsa.add (Spec.MlDsa.polyAt s.mem (pa s h)) (Spec.MlDsa.multiplyNTT (Spec.MlDsa.polyAt s.mem (pa s f)) (Spec.MlDsa.polyAt s.mem (pa s g)))) := by obtain ⟨i1, i2, _⟩ := sepB_spec h1 obtain ⟨_, i3, _⟩ := sepB_spec h2 @@ -442,9 +442,9 @@ theorem mulAddAt_ok {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulAddCo rwa [ce_polyAt (by rw [hsp]; exact L.stkD i1), ce_polyAt (by rw [hsp]; exact L.stkD i2), ce_polyAt (by rw [hsp]; exact L.stkD i3), hm] at hpost include hh hf hg h1 h2 w1 in -theorem mulAddAt_tr {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulAddContract X86_64.abi stk) +theorem mulAddAt_tr {sfx : String} {c : Prog isa} (hc : Callee c fun stk => Spec.MlDsa.mulAddContract X86_64.abi stk) (hbh : h.1 ∈ kgRegs) (hbf : f.1 ∈ kgRegs) (hbg : g.1 ∈ kgRegs) : - RelCT isa (fun x y => Two p x y ∧ (Spec.MlDsa.Reduced x.mem (pa x h) ∧ Spec.MlDsa.Reduced x.mem (pa x f) ∧ Spec.MlDsa.Reduced x.mem (pa x g)) ∧ (Spec.MlDsa.Reduced y.mem (pa y h) ∧ Spec.MlDsa.Reduced y.mem (pa y f) ∧ Spec.MlDsa.Reduced y.mem (pa y g))) (mulAddAt c h f g) fun _ _ => True := by + RelCT isa (fun x y => Two p x y ∧ (Spec.MlDsa.Reduced x.mem (pa x h) ∧ Spec.MlDsa.Reduced x.mem (pa x f) ∧ Spec.MlDsa.Reduced x.mem (pa x g)) ∧ (Spec.MlDsa.Reduced y.mem (pa y h) ∧ Spec.MlDsa.Reduced y.mem (pa y f) ∧ Spec.MlDsa.Reduced y.mem (pa y g))) (mulAddAt sfx c h f g) fun _ _ => True := by obtain ⟨i1, i2, _⟩ := sepB_spec h1 obtain ⟨_, i3, _⟩ := sepB_spec h2 refine primTr hc (nomem_append (nomem_append (lea_nomem _ _) (lea_nomem _ _)) (lea_nomem _ _)) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean index 245ada812..0713c4074 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean @@ -9,14 +9,17 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejBoundedCT import VerifiedGarbage.Proof.MlDsa.X86_64.Round.Power2Round import VerifiedGarbage.Proof.MlDsa.X86_64.Pack.SimpleBitPack import VerifiedGarbage.Proof.MlDsa.X86_64.Pack.BitPack +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Backend +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Same /-! # ML-DSA key generation on x86-64, with this library's primitives Untrusted: everything here is checked by Lean. The x86-64 implementations of the primitives (`prims`) are verified, use at most 16 bytes of stack, and -never write `rsp` but by calls nested at most twice (`prims_ok`), so key -generation with them is verified (`keyGen44_verified`, …). +never write `rsp` but by calls nested at most twice, with any implementation +`v` of the polynomial arithmetic (`prims_okWith`), so key generation with +them is verified (`keyGen_verifiedWith`). -/ namespace VG.Proof.MlDsa.X86_64.KeyGen @@ -24,29 +27,56 @@ namespace VG.Proof.MlDsa.X86_64.KeyGen open VG VG.X86_64 VG.Proof.MlKem.X86_64 open VG.Impl.MlDsa.X86_64.KeyGen -theorem prims_ok : PrimsOk prims where - ntt := ⟨⟨0, by decide, Arith.ntt_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ - invNtt := ⟨⟨0, by decide, Arith.nttInv_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ - mul := ⟨⟨0, by decide, Arith.mul_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ - mulAdd := ⟨⟨0, by decide, Arith.mulAdd_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ - add := ⟨⟨0, by decide, Arith.add_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ - rejNtt := ⟨⟨16, by decide, Sample.rejNTT_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ - rejBounded := ⟨⟨16, by decide, Sample.rejBounded_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ - power2Round := ⟨⟨0, by decide, Round.power2Round_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ - simpleBitPack := ⟨⟨0, by decide, Pack.simpleBitPack_verified⟩, nosp_of (by decide +kernel), - by decide +kernel⟩ - bitPack := ⟨⟨0, by decide, Pack.bitPack_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ - -theorem keyGen44_verified : - Verified X86_64.target keyGen44 (Spec.MlDsa.keyGenContract Spec.MlDsa.mlDsa44 X86_64.abi 32) := - keyGen_verified prims_ok _ (.inl rfl) - -theorem keyGen65_verified : - Verified X86_64.target keyGen65 (Spec.MlDsa.keyGenContract Spec.MlDsa.mlDsa65 X86_64.abi 32) := - keyGen_verified prims_ok _ (.inr (.inl rfl)) - -theorem keyGen87_verified : - Verified X86_64.target keyGen87 (Spec.MlDsa.keyGenContract Spec.MlDsa.mlDsa87 X86_64.abi 32) := - keyGen_verified prims_ok _ (.inr (.inr rfl)) +open VG.Proof.MlDsa.X86_64 (FnOk ArithImpl Comp Same Same.ok Code.allInstrs_of_all) +open VG.Impl.MlDsa.X86_64.Arith (Backend) + +/-- A function of the polynomial arithmetic satisfies what the proofs of key generation need of it. -/ +theorem calleeOf {k : Nat → Contract isa} {c : Prog isa} (h : FnOk k c) : Callee c k := + ⟨⟨0, by decide, h.ver⟩, h.nosp, h.depth⟩ + +theorem prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) where + ntt := calleeOf v.ok.ntt + invNtt := calleeOf v.ok.invNtt + mul := calleeOf v.ok.mul + mulAdd := calleeOf v.ok.mulAdd + add := calleeOf v.ok.add + rejNtt := (⟨⟨16, by decide, Sample.rejNTT_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ : + Callee prims.rejNtt _) + rejBounded := (⟨⟨16, by decide, Sample.rejBounded_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ : + Callee prims.rejBounded _) + power2Round := (⟨⟨0, by decide, Round.power2Round_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ : + Callee prims.power2Round _) + simpleBitPack := (⟨⟨0, by decide, Pack.simpleBitPack_verified⟩, nosp_of (by decide +kernel), + by decide +kernel⟩ : Callee prims.simpleBitPack _) + bitPack := (⟨⟨0, by decide, Pack.bitPack_verified⟩, nosp_of (by decide +kernel), by decide +kernel⟩ : + Callee prims.bitPack _) + +/-! For any implementation of the polynomial arithmetic, that key +generation never writes `rsp` is checked by evaluating it with every +function of it empty (`keyGen_same`, as `sign_same`). -/ + +theorem keyGen_same {m mc : Prog isa → Bool} (hm : Comp m mc) {B : Backend} (h1 : mc B.ntt = true) + (h2 : mc B.invNtt = true) (h3 : mc B.mul = true) (h4 : mc B.mulAdd = true) (h5 : mc B.add = true) + (p : Spec.MlDsa.Params) : Same m (keyGen (primsWith B) p) (keyGen (primsWith .empty) p) := by + unfold keyGen + same_tac hm + +theorem keyGen0_sp {p : Spec.MlDsa.Params} + (hp : p = Spec.MlDsa.mlDsa44 ∨ p = Spec.MlDsa.mlDsa65 ∨ p = Spec.MlDsa.mlDsa87) : + (keyGen (primsWith .empty) p).allInstrs (fun i => !isa.writesSp i) = true := by + rcases hp with rfl | rfl | rfl <;> decide +kernel + +variable (v : ArithImpl) {p : Spec.MlDsa.Params} + (hp : p = Spec.MlDsa.mlDsa44 ∨ p = Spec.MlDsa.mlDsa65 ∨ p = Spec.MlDsa.mlDsa87) +include hp + +theorem keyGen_spSafe : (keyGen (primsWith v.code) p).all (fun i => !isa.writesSp i) = true := + Code.all_of_allInstrs (Same.ok (keyGen_same (Comp.all _) (Code.allInstrs_of_all v.ok.ntt.sp) + (Code.allInstrs_of_all v.ok.invNtt.sp) (Code.allInstrs_of_all v.ok.mul.sp) + (Code.allInstrs_of_all v.ok.mulAdd.sp) (Code.allInstrs_of_all v.ok.add.sp) p) (keyGen0_sp hp)) + +theorem keyGen_verifiedWith : + Verified X86_64.target (keyGen (primsWith v.code) p) (Spec.MlDsa.keyGenContract p X86_64.abi 32) := + keyGen_verified (prims_okWith v) p hp end VG.Proof.MlDsa.X86_64.KeyGen diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestRow.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestRow.lean index c1ff8245b..636a4b89d 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestRow.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/RestRow.lean @@ -75,7 +75,7 @@ include hP hF hp hi /-- `t = Â[i, 0] ŝ₁[0]`. -/ theorem mul_ok {A : Nat → Poly} {S : Nat → IPoly} {R : BitVec 64} {s : State} (h : KR p σ A S R (p.ℓ + p.k) p.ℓ i s) : - WP isa (mulAt P.mul (tP p) (aP (p.ℓ * i)) (sP p 0)) s fun s' => + WP isa (mulAt P.sfx P.mul (tP p) (aP (p.ℓ * i)) (sP p 0)) s fun s' => KR p σ A S R (p.ℓ + p.k) p.ℓ i s' ∧ tIs p (fun A S => dotK p A S i 1) A S s' := by have hkl := hF.kl; have hl := hF.l; have hk := hF.k have hx0 := idx_lt (j := 0) hi (by omega) @@ -94,7 +94,7 @@ theorem mul_ok {A : Nat → Poly} {S : Nat → IPoly} {R : BitVec 64} {s : State /-- `t = t + Â[i, j] ŝ₁[j]`. -/ theorem mulAdd_ok {j : Nat} (hj : j < p.ℓ) {A : Nat → Poly} {S : Nat → IPoly} {R : BitVec 64} {s : State} (h : KR p σ A S R (p.ℓ + p.k) p.ℓ i s) (ht : tIs p (fun A S => dotK p A S i j) A S s) : - WP isa (mulAddAt P.mulAdd (tP p) (aP (p.ℓ * i + j)) (sP p j)) s fun s' => + WP isa (mulAddAt P.sfx P.mulAdd (tP p) (aP (p.ℓ * i + j)) (sP p j)) s fun s' => KR p σ A S R (p.ℓ + p.k) p.ℓ i s' ∧ tIs p (fun A S => dotK p A S i (j + 1)) A S s' := by have hkl := hF.kl; have hl := hF.l; have hk := hF.k dsimp only [tIs] at ht @@ -112,7 +112,7 @@ theorem mulAdd_ok {j : Nat} (hj : j < p.ℓ) {A : Nat → Poly} {S : Nat → IPo /-- `t = NTT⁻¹(t)`. -/ theorem inv_ok {A : Nat → Poly} {S : Nat → IPoly} {R : BitVec 64} {s : State} (h : KR p σ A S R (p.ℓ + p.k) p.ℓ i s) (ht : tIs p (fun A S => dotK p A S i p.ℓ) A S s) : - WP isa (invNttAt P.invNtt (tP p)) s fun s' => + WP isa (invNttAt P.sfx P.invNtt (tP p)) s fun s' => KR p σ A S R (p.ℓ + p.k) p.ℓ i s' ∧ tIs p (fun A S => nttInv (dotK p A S i p.ℓ)) A S s' := by have hkl := hF.kl; have hl := hF.l; have hk := hF.k dsimp only [tIs] at ht @@ -128,7 +128,7 @@ theorem inv_ok {A : Nat → Poly} {S : Nat → IPoly} {R : BitVec 64} {s : State /-- `t = t + s₂[i]`. -/ theorem addS2_ok {A : Nat → Poly} {S : Nat → IPoly} {R : BitVec 64} {s : State} (h : KR p σ A S R (p.ℓ + p.k) p.ℓ i s) (ht : tIs p (fun A S => nttInv (dotK p A S i p.ℓ)) A S s) : - WP isa (addAt P.add (tP p) (sP p (p.ℓ + i))) s fun s' => + WP isa (addAt P.sfx P.add (tP p) (sP p (p.ℓ + i))) s fun s' => KR p σ A S R (p.ℓ + p.k) p.ℓ i s' ∧ tIs p (fun A S => tK p A S i) A S s' := by have hkl := hF.kl; have hl := hF.l; have hk := hF.k dsimp only [tIs] at ht @@ -226,7 +226,7 @@ variable {P : Prims} (hP : PrimsOk P) {p : Params} (hF : PFacts p) {i : Nat} (hi include hP hF hi theorem mul_piece : Piece p (KRx p (p.ℓ + p.k) p.ℓ i) (RowI p i (tIs p fun A S => dotK p A S i 1)) - (mulAt P.mul (tP p) (aP (p.ℓ * i)) (sP p 0)) := by + (mulAt P.sfx P.mul (tP p) (aP (p.ℓ * i)) (sP p 0)) := by have hkl := hF.kl; have hl := hF.l; have hk := hF.k have hx0 := idx_lt (j := 0) hi (by omega) rw [Nat.add_zero] at hx0 @@ -244,7 +244,7 @@ theorem mul_piece : Piece p (KRx p (p.ℓ + p.k) p.ℓ i) (RowI p i (tIs p fun A theorem mulAdd_piece {j : Nat} (hj : j < p.ℓ) : Piece p (RowI p i (tIs p fun A S => dotK p A S i j)) (RowI p i (tIs p fun A S => dotK p A S i (j + 1))) - (mulAddAt P.mulAdd (tP p) (aP (p.ℓ * i + j)) (sP p j)) := by + (mulAddAt P.sfx P.mulAdd (tP p) (aP (p.ℓ * i + j)) (sP p j)) := by have hkl := hF.kl; have hl := hF.l; have hk := hF.k have hx0 := idx_lt hi hj refine ⟨fun _ _ hp ⟨A, S, R, h, ht⟩ => WP.mono (mulAdd_ok hP hF hp hi hj h ht) fun _ h => ⟨A, S, R, h⟩, ?_⟩ @@ -259,7 +259,7 @@ theorem mulAdd_piece {j : Nat} (hj : j < p.ℓ) : (show Reg.rbx ∈ kgRegs by decide) theorem inv_piece : Piece p (RowI p i (tIs p fun A S => dotK p A S i p.ℓ)) - (RowI p i (tIs p fun A S => nttInv (dotK p A S i p.ℓ))) (invNttAt P.invNtt (tP p)) := by + (RowI p i (tIs p fun A S => nttInv (dotK p A S i p.ℓ))) (invNttAt P.sfx P.invNtt (tP p)) := by have hkl := hF.kl; have hl := hF.l; have hk := hF.k refine ⟨fun _ _ hp ⟨A, S, R, h, ht⟩ => WP.mono (inv_ok hP hF hp hi h ht) fun _ h => ⟨A, S, R, h⟩, ?_⟩ refine rel_of (Q := fun x y => Two p x y ∧ Reduced x.mem (pa x (tP p)) ∧ Reduced y.mem (pa y (tP p))) ?_ @@ -268,7 +268,7 @@ theorem inv_piece : Piece p (RowI p i (tIs p fun A S => dotK p A S i p.ℓ)) exact ipAt_tr (tP_ok hF) (by lay) (by lay) (by lay) hP.invNtt (show Reg.rbx ∈ kgRegs by decide) theorem addS2_piece : Piece p (RowI p i (tIs p fun A S => nttInv (dotK p A S i p.ℓ))) - (RowI p i (tIs p fun A S => Proof.MlDsa.KeyGen.tK p A S i)) (addAt P.add (tP p) (sP p (p.ℓ + i))) := by + (RowI p i (tIs p fun A S => Proof.MlDsa.KeyGen.tK p A S i)) (addAt P.sfx P.add (tP p) (sP p (p.ℓ + i))) := by have hkl := hF.kl; have hl := hF.l; have hk := hF.k refine ⟨fun _ _ hp ⟨A, S, R, h, ht⟩ => WP.mono (addS2_ok hP hF hp hi h ht) fun _ h => ⟨A, S, R, h⟩, ?_⟩ refine rel_of (Q := fun x y => Two p x y ∧ (Reduced x.mem (pa x (tP p)) ∧ Reduced x.mem (pa x (sP p (p.ℓ + i)))) ∧ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean index f88d2d5a0..0d9b4dcd2 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean @@ -11,13 +11,15 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Pack.HintPack import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejNttCT import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.ExpandMask import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.BallCT +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Backend /-! # ML-DSA signing on x86-64: the primitives it calls Untrusted: everything here is checked by Lean. The verified x86-64 implementations of the primitives (`prims`), and what the proofs of signing -need of them (`prims_ok`), with 24 bytes of stack for each call: their +need of them, with any implementation `v` of the polynomial arithmetic +(`prims_okWith`), with 24 bytes of stack for each call: their contracts, and, of the two samplers whose result signing branches on, that it depends only on their public data and that they succeed only if the algorithm finishes within `maxBounds` (from what their own proofs say they @@ -30,6 +32,7 @@ open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Sign open VG.Proof.MlDsa.Sign open VG.Spec.MlDsa open VG.Spec.Sha3 (bytesAt) +open VG.Proof.MlDsa.X86_64 (FnOk ArithImpl) /-- The x86-64 implementations of the primitives. -/ def prims : Prims where @@ -51,6 +54,17 @@ def prims : Prims where bitUnpack := Impl.MlDsa.X86_64.Pack.bitUnpack hintBitPack := Impl.MlDsa.X86_64.Pack.hintBitPack +/-- The primitives, with the polynomial arithmetic of `B`. -/ +def primsWith (B : Impl.MlDsa.X86_64.Arith.Backend) : Prims := + { prims with + ntt := B.ntt + invNtt := B.invNtt + mul := B.mul + mulAdd := B.mulAdd + add := B.add + sub := B.sub + sfx := B.sfx } + theorem nosp_of {c : Prog isa} (h : c.allInstrs (fun i => !Taint.clobbers i .rsp) = true) : NoSp c := by rw [Code.allInstrs_eq] at h intro i hi @@ -105,31 +119,47 @@ end theorem one_ne_zero32 : (1 : BitVec 32) ≠ 0 := by decide -/-- The primitives satisfy what the proofs of signing need of them. -/ -def prims_ok : PrimsOk prims signStack where - ntt := ⟨0, by decide, Proof.MlDsa.X86_64.Arith.ntt_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - invNtt := ⟨0, by decide, Proof.MlDsa.X86_64.Arith.nttInv_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - mul := ⟨0, by decide, Proof.MlDsa.X86_64.Arith.mul_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - mulAdd := ⟨0, by decide, Proof.MlDsa.X86_64.Arith.mulAdd_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - add := ⟨0, by decide, Proof.MlDsa.X86_64.Arith.add_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - sub := ⟨0, by decide, Proof.MlDsa.X86_64.Arith.sub_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - rejNTT := ⟨16, by decide, Proof.MlDsa.X86_64.Sample.rejNTT_verified, nosp_of (by decide +kernel), - by decide +kernel⟩ - expandMask := ⟨16, by decide, Proof.MlDsa.X86_64.Sample.expandMask_verified, nosp_of (by decide +kernel), - by decide +kernel⟩ - ball := ⟨16, by decide, Proof.MlDsa.X86_64.Sample.sampleInBall_verified, nosp_of (by decide +kernel), - by decide +kernel⟩ - highBits := ⟨0, by decide, Proof.MlDsa.X86_64.Round.highBits_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - lowBits := ⟨0, by decide, Proof.MlDsa.X86_64.Round.lowBits_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - normLt := ⟨0, by decide, Proof.MlDsa.X86_64.Round.normLt_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - makeHint := ⟨0, by decide, Proof.MlDsa.X86_64.Round.makeHint_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - simpleBitPack := ⟨0, by decide, Proof.MlDsa.X86_64.Pack.simpleBitPack_verified, nosp_of (by decide +kernel), - by decide +kernel⟩ - bitPack := ⟨0, by decide, Proof.MlDsa.X86_64.Pack.bitPack_verified, nosp_of (by decide +kernel), by decide +kernel⟩ - bitUnpack := ⟨0, by decide, Proof.MlDsa.X86_64.Pack.bitUnpack_verified, nosp_of (by decide +kernel), - by decide +kernel⟩ - hintBitPack := ⟨0, by decide, Proof.MlDsa.X86_64.Pack.hintBitPack_verified, nosp_of (by decide +kernel), - by decide +kernel⟩ +/-- A function of the polynomial arithmetic satisfies what the proofs of signing need of it. -/ +def calleeOf {k : Nat → Contract isa} {c : Prog isa} (h : FnOk k c) : Callee k signStack c := + ⟨0, by decide, h.ver, h.nosp, by have := h.depth; unfold signStack; omega⟩ + +/-- The primitives, with the polynomial arithmetic of `v`, satisfy what the +proofs of signing need of them. -/ +def prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) signStack where + ntt := calleeOf v.ok.ntt + invNtt := calleeOf v.ok.invNtt + mul := calleeOf v.ok.mul + mulAdd := calleeOf v.ok.mulAdd + add := calleeOf v.ok.add + sub := calleeOf v.ok.sub + rejNTT := (⟨16, by decide, Proof.MlDsa.X86_64.Sample.rejNTT_verified, nosp_of (by decide +kernel), + by decide +kernel⟩ : + Callee _ signStack prims.rejNTT) + expandMask := (⟨16, by decide, Proof.MlDsa.X86_64.Sample.expandMask_verified, nosp_of (by decide +kernel), + by decide +kernel⟩ : + Callee _ signStack prims.expandMask) + ball := (⟨16, by decide, Proof.MlDsa.X86_64.Sample.sampleInBall_verified, nosp_of (by decide +kernel), + by decide +kernel⟩ : + Callee _ signStack prims.ball) + highBits := (⟨0, by decide, Proof.MlDsa.X86_64.Round.highBits_verified, nosp_of (by decide +kernel), by decide +kernel⟩ : + Callee _ signStack prims.highBits) + lowBits := (⟨0, by decide, Proof.MlDsa.X86_64.Round.lowBits_verified, nosp_of (by decide +kernel), by decide +kernel⟩ : + Callee _ signStack prims.lowBits) + normLt := (⟨0, by decide, Proof.MlDsa.X86_64.Round.normLt_verified, nosp_of (by decide +kernel), by decide +kernel⟩ : + Callee _ signStack prims.normLt) + makeHint := (⟨0, by decide, Proof.MlDsa.X86_64.Round.makeHint_verified, nosp_of (by decide +kernel), by decide +kernel⟩ : + Callee _ signStack prims.makeHint) + simpleBitPack := (⟨0, by decide, Proof.MlDsa.X86_64.Pack.simpleBitPack_verified, nosp_of (by decide +kernel), + by decide +kernel⟩ : + Callee _ signStack prims.simpleBitPack) + bitPack := (⟨0, by decide, Proof.MlDsa.X86_64.Pack.bitPack_verified, nosp_of (by decide +kernel), by decide +kernel⟩ : + Callee _ signStack prims.bitPack) + bitUnpack := (⟨0, by decide, Proof.MlDsa.X86_64.Pack.bitUnpack_verified, nosp_of (by decide +kernel), + by decide +kernel⟩ : + Callee _ signStack prims.bitUnpack) + hintBitPack := (⟨0, by decide, Proof.MlDsa.X86_64.Pack.hintBitPack_verified, nosp_of (by decide +kernel), + by decide +kernel⟩ : + Callee _ signStack prims.hintBitPack) rejRet := fun s₁ s₂ t₁ t₂ s₁' s₂' ⟨h₁, h₂, hp⟩ e₁ e₂ => ⟨Proof.MlDsa.X86_64.Sample.rejNTT_verified.2.1 s₁ s₂ t₁ t₂ s₁' s₂' h₁ h₂ hp e₁ e₂, show _ = _ by rw [rn_ret h₁ e₁, rn_ret h₂ e₂, rn_pub s₁ s₂ hp]⟩ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean index 089924f6d..3b2dc921e 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean @@ -1,10 +1,12 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.Inst +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Same /-! # ML-DSA signing on x86-64: verified Untrusted: everything here is checked by Lean. `vg_mldsa{44,65,87}_sign` -(`sign prims p`) is verified against `signContractT`: `signContract` with +(`sign (primsWith v.code) p`, for an implementation `v` of the polynomial +arithmetic) is verified against `signContractT`: `signContract` with `signLeakT` (`Proof/MlDsa/Sign/Leak.lean`) for `signLeak`, which tags what each iteration of the loop leaks after its `c̃` with whether it was rejected. The contract's `signLeak` tags the iterations the same way @@ -18,6 +20,8 @@ open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Sign open VG.Proof.MlDsa.Sign open VG.Spec.MlDsa open VG.Spec.Sha3 (bytesAt) +open VG.Proof.MlDsa.X86_64 (Comp Same Same.ok ArithImpl Code.allInstrs_of_all) +open VG.Impl.MlDsa.X86_64.Arith (Backend) /-- `signContract`, with `signLeakT` for `signLeak`. -/ def signContractT (p : Params) {M : ISA} (A : Abi M) (stack : Nat := 0) : Contract M := @@ -62,52 +66,56 @@ theorem signK_implies {p : Params} (h3 : Ok3 p) : · sig_implies_sat [signContractT, signSig, X86_64.abi, X86_64.argRegs] [signSat] using signSat mlDsa65 · sig_implies_sat [signContractT, signSig, X86_64.abi, X86_64.argRegs] [signSat] using signSat mlDsa87 -theorem sign_verified {p : Params} (h3 : Ok3 p) - (hmx : ctlOk (Impl.MlDsa.X86_64.Sign.sign prims p) = true) : - Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign prims p) (signContractT p X86_64.abi signStack) := - Verified.of_correct (sign_correct prims_ok h3 hmx) (sign_ct prims_ok h3) (signK_implies h3) +/-! ## For any implementation of the polynomial arithmetic -theorem sign44_verified : - Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign prims mlDsa44) (signContractT mlDsa44 X86_64.abi signStack) := - sign_verified (.inl rfl) (by decide +kernel) +A check that composes over the code (`Comp`) gives the same result on +signing with the polynomial arithmetic `B` as with every function of it +empty, if it holds of `B`'s functions (`sign_same`), so MXCSR (`sign_ctl`) +and the stack pointer (`sign_spSafe`) are checked by evaluating signing +with no implementation of it. -/ -theorem sign65_verified : - Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign prims mlDsa65) (signContractT mlDsa65 X86_64.abi signStack) := - sign_verified (.inr (.inl rfl)) (by decide +kernel) +theorem sign_same {m mc : Prog isa → Bool} (hm : Comp m mc) {B : Backend} (h1 : mc B.ntt = true) + (h2 : mc B.invNtt = true) (h3 : mc B.mul = true) (h4 : mc B.mulAdd = true) (h5 : mc B.add = true) + (h6 : mc B.sub = true) (p : Params) : + Same m (Impl.MlDsa.X86_64.Sign.sign (primsWith B) p) (Impl.MlDsa.X86_64.Sign.sign (primsWith .empty) p) := by + unfold Impl.MlDsa.X86_64.Sign.sign + same_tac hm -theorem sign87_verified : - Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign prims mlDsa87) (signContractT mlDsa87 X86_64.abi signStack) := - sign_verified (.inr (.inr rfl)) (by decide +kernel) +theorem sign0_ctlC {p : Params} (h3 : Ok3 p) : ctlC (Impl.MlDsa.X86_64.Sign.sign (primsWith .empty) p) = true := by + rcases h3 with rfl | rfl | rfl <;> decide +kernel -/-! Against the contract: `signContractT` is `signContract`, whose leakage -tags each iteration as `signLeakT` does (`signLeakT_eq_signLeak`). -/ - -theorem signContractT_eq (p : Params) {M : ISA} (A : Abi M) (stack : Nat) : - signContractT p A stack = signContract p A stack := by - unfold signContractT signContract - simp only [Sign.signLeakT_eq_signLeak] +theorem sign0_sp {p : Params} (h3 : Ok3 p) : + (Impl.MlDsa.X86_64.Sign.sign (primsWith .empty) p).allInstrs (fun i => !isa.writesSp i) = true := by + rcases h3 with rfl | rfl | rfl <;> decide +kernel -theorem sign44_verified' : - Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign prims mlDsa44) (signContract mlDsa44 X86_64.abi signStack) := - signContractT_eq mlDsa44 X86_64.abi signStack ▸ sign44_verified +variable (v : ArithImpl) {p : Params} (h3 : Ok3 p) +include h3 -theorem sign65_verified' : - Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign prims mlDsa65) (signContract mlDsa65 X86_64.abi signStack) := - signContractT_eq mlDsa65 X86_64.abi signStack ▸ sign65_verified +theorem sign_ctl : ctlOk (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p) = true := + ctlOk_of_ctlC (Same.ok (sign_same Comp.ctlC v.ok.ntt.ctl v.ok.invNtt.ctl v.ok.mul.ctl v.ok.mulAdd.ctl + v.ok.add.ctl v.ok.sub.ctl p) (sign0_ctlC h3)) -theorem sign87_verified' : - Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign prims mlDsa87) (signContract mlDsa87 X86_64.abi signStack) := - signContractT_eq mlDsa87 X86_64.abi signStack ▸ sign87_verified +theorem sign_spSafe : (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p).all (fun i => !isa.writesSp i) = true := + Code.all_of_allInstrs (Same.ok (sign_same (Comp.all _) (Code.allInstrs_of_all v.ok.ntt.sp) + (Code.allInstrs_of_all v.ok.invNtt.sp) (Code.allInstrs_of_all v.ok.mul.sp) + (Code.allInstrs_of_all v.ok.mulAdd.sp) (Code.allInstrs_of_all v.ok.add.sp) + (Code.allInstrs_of_all v.ok.sub.sp) p) (sign0_sp h3)) -/-! What registering them needs of the code besides: it never writes the stack pointer. -/ +theorem sign_verified : + Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p) (signContractT p X86_64.abi signStack) := + Verified.of_correct (sign_correct (prims_okWith v) h3 (sign_ctl v h3)) (sign_ct (prims_okWith v) h3) + (signK_implies h3) -theorem sign44_spSafe : (Impl.MlDsa.X86_64.Sign.sign prims mlDsa44).all (fun i => !X86_64.target.isa.writesSp i) = true := - Code.all_of_allInstrs (by decide +kernel) - -theorem sign65_spSafe : (Impl.MlDsa.X86_64.Sign.sign prims mlDsa65).all (fun i => !X86_64.target.isa.writesSp i) = true := - Code.all_of_allInstrs (by decide +kernel) +omit h3 in +/-- Against the contract: `signContractT` is `signContract`, whose leakage +tags each iteration as `signLeakT` does (`signLeakT_eq_signLeak`). -/ +theorem signContractT_eq (p : Params) {M : ISA} (A : Abi M) (stack : Nat) : + signContractT p A stack = signContract p A stack := by + unfold signContractT signContract + simp only [Sign.signLeakT_eq_signLeak] -theorem sign87_spSafe : (Impl.MlDsa.X86_64.Sign.sign prims mlDsa87).all (fun i => !X86_64.target.isa.writesSp i) = true := - Code.all_of_allInstrs (by decide +kernel) +theorem sign_verified' : + Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p) (signContract p X86_64.abi signStack) := + signContractT_eq p X86_64.abi signStack ▸ sign_verified v h3 end VG.Proof.MlDsa.X86_64.Sign diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean index ce8c82e43..852e3e7a1 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean @@ -1,5 +1,6 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Verify.PrimsOk import VerifiedGarbage.Impl.MlDsa.X86_64.Verify.Verify +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Same /-! # ML-DSA verification on x86-64: properties of every instruction @@ -20,7 +21,7 @@ open VG.Spec.MlDsa /-- The primitives, each empty. -/ def P0 : Prims := ⟨.block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], - .block [], .block [], .block [], .block []⟩ + .block [], .block [], .block [], .block [], ""⟩ /-- `q` holds of every instruction of the primitives `P`. -/ structure PrimsQ (q : Instr → Bool) (P : Prims) : Prop where @@ -49,8 +50,8 @@ theorem SameQ.seq {a a' b b' : Prog isa} (ha : SameQ q a a') (hb : SameQ q b b') show (a.allInstrs q && b.allInstrs q) = (a'.allInstrs q && b'.allInstrs q) rw [show a.allInstrs q = a'.allInstrs q from ha, show b.allInstrs q = b'.allInstrs q from hb] -theorem SameQ.call {c : Prog isa} (hc : c.allInstrs q = true) (n : String) (as : List (Reg × Arg)) : - SameQ q (callAt n c as) (callAt n (.block []) as) := by +theorem SameQ.call {c : Prog isa} (hc : c.allInstrs q = true) {n n' : String} (as : List (Reg × Arg)) : + SameQ q (callAt n c as) (callAt n' (.block []) as) := by show (_ && c.allInstrs q) = (_ && true) rw [hc] @@ -70,30 +71,30 @@ theorem SameQ.sampled {c c' : Prog isa} (h : SameQ q c c') (a : Ptr) : SameQ q ( variable {P : Prims} (hP : PrimsQ q P) (p : Params) include hP -theorem hint_q : SameQ q (hint P p) (hint P0 p) := (SameQ.call hP.hintUnpack _ _).seq rfl +theorem hint_q : SameQ q (hint P p) (hint P0 p) := (SameQ.call hP.hintUnpack _).seq rfl theorem zOne_q (i : Nat) : SameQ q (zOne P p i) (zOne P0 p i) := - (SameQ.call hP.bitUnpack _ _).seq ((SameQ.call hP.normLt _ _).seq rfl) + (SameQ.call hP.bitUnpack _).seq ((SameQ.call hP.normLt _).seq rfl) omit p in theorem aOne_q (e : Nat) : SameQ q (aOne P e) (aOne P0 e) := - SameQ.seq rfl (SameQ.sampled (SameQ.call hP.rejNtt _ _) _) + SameQ.seq rfl (SameQ.sampled (SameQ.call hP.rejNtt _) _) theorem aRow_q (r : Nat) : SameQ q (aRow P p r) (aRow P0 p r) := SameQ.seqR (aOne_q hP) _ _ theorem samples_q : SameQ q (samples P p) (samples P0 p) := - SameQ.seq rfl ((SameQ.seqR (aRow_q hP p) _ _).seq (SameQ.sampled (SameQ.call hP.ball _ _) _)) + SameQ.seq rfl ((SameQ.seqR (aRow_q hP p) _ _).seq (SameQ.sampled (SameQ.call hP.ball _) _)) theorem dot_q (r : Nat) : SameQ q (dot P p r) (dot P0 p r) := - (SameQ.call hP.mul _ _).seq (SameQ.seqR (fun _ => SameQ.call hP.mulAdd _ _) _ _) + (SameQ.call hP.mul _).seq (SameQ.seqR (fun _ => SameQ.call hP.mulAdd _) _ _) theorem row_q (r : Nat) : SameQ q (row P p r) (row P0 p r) := - (dot_q hP p r).seq ((SameQ.call hP.unpackT1 _ _).seq ((SameQ.call hP.ntt _ _).seq ((SameQ.call hP.mul _ _).seq - ((SameQ.call hP.sub _ _).seq ((SameQ.call hP.invNtt _ _).seq ((SameQ.call hP.useHint _ _).seq - (SameQ.call hP.simpleBitPack _ _))))))) + (dot_q hP p r).seq ((SameQ.call hP.unpackT1 _).seq ((SameQ.call hP.ntt _).seq ((SameQ.call hP.mul _).seq + ((SameQ.call hP.sub _).seq ((SameQ.call hP.invNtt _).seq ((SameQ.call hP.useHint _).seq + (SameQ.call hP.simpleBitPack _))))))) theorem compute_q : SameQ q (compute P p) (compute P0 p) := - (SameQ.seqR (fun _ => SameQ.call hP.ntt _ _) _ _).seq ((SameQ.call hP.ntt _ _).seq + (SameQ.seqR (fun _ => SameQ.call hP.ntt _) _ _).seq ((SameQ.call hP.ntt _).seq ((SameQ.seqR (row_q hP p) _ _).seq rfl)) theorem verify_q : SameQ q (verify P p) (verify P0 p) := @@ -102,12 +103,6 @@ theorem verify_q : SameQ q (verify P p) (verify P0 p) := end -theorem Code.allInstrs_of_all {I C : Type} {q : I → Bool} {c : Code I C} (h : c.all q = true) : - c.allInstrs q = true := by - induction c with - | block is => induction is <;> simp_all [Code.all, Code.allInstrs] - | _ => simp_all [Code.all, Code.allInstrs] - theorem verify0_sp : ∀ p ∈ params, (verify P0 p).allInstrs (fun i => !isa.writesSp i) = true := by decide +kernel @@ -136,8 +131,8 @@ theorem SameC.seq {a a' b b' : Prog isa} (ha : SameC a a') (hb : SameC b b') : S show (ctlC a && ctlC b) = (ctlC a' && ctlC b') rw [show ctlC a = ctlC a' from ha, show ctlC b = ctlC b' from hb] -theorem SameC.call {c : Prog isa} (hc : ctlOk c = true) (n : String) (as : List (Reg × Arg)) : - SameC (callAt n c as) (callAt n (.block []) as) := by +theorem SameC.call {c : Prog isa} (hc : ctlOk c = true) {n n' : String} (as : List (Reg × Arg)) : + SameC (callAt n c as) (callAt n' (.block []) as) := by show (_ && ctlOk c) = (_ && true) rw [hc] @@ -159,22 +154,22 @@ include hP theorem verify_c : SameC (verify P p) (verify P0 p) := by have aOne : ∀ e, SameC (aOne P e) (aOne P0 e) := fun e => - SameC.seq rfl (SameC.sampled (SameC.call hP.rejNtt _ _) _) + SameC.seq rfl (SameC.sampled (SameC.call hP.rejNtt _) _) have dot : ∀ r, SameC (dot P p r) (dot P0 p r) := fun r => - (SameC.call hP.mul _ _).seq (SameC.seqR (fun _ => SameC.call hP.mulAdd _ _) _ _) + (SameC.call hP.mul _).seq (SameC.seqR (fun _ => SameC.call hP.mulAdd _) _ _) have row : ∀ r, SameC (row P p r) (row P0 p r) := fun r => - (dot r).seq ((SameC.call hP.unpackT1 _ _).seq ((SameC.call hP.ntt _ _).seq ((SameC.call hP.mul _ _).seq - ((SameC.call hP.sub _ _).seq ((SameC.call hP.invNtt _ _).seq ((SameC.call hP.useHint _ _).seq - (SameC.call hP.simpleBitPack _ _))))))) + (dot r).seq ((SameC.call hP.unpackT1 _).seq ((SameC.call hP.ntt _).seq ((SameC.call hP.mul _).seq + ((SameC.call hP.sub _).seq ((SameC.call hP.invNtt _).seq ((SameC.call hP.useHint _).seq + (SameC.call hP.simpleBitPack _))))))) have samples : SameC (samples P p) (samples P0 p) := SameC.seq rfl ((SameC.seqR (fun r => SameC.seqR aOne _ _) _ _).seq - (SameC.sampled (SameC.call hP.ball _ _) _)) + (SameC.sampled (SameC.call hP.ball _) _)) have compute : SameC (compute P p) (compute P0 p) := - (SameC.seqR (fun _ => SameC.call hP.ntt _ _) _ _).seq ((SameC.call hP.ntt _ _).seq + (SameC.seqR (fun _ => SameC.call hP.ntt _) _ _).seq ((SameC.call hP.ntt _).seq ((SameC.seqR row _ _).seq rfl)) have zOne : ∀ i, SameC (zOne P p i) (zOne P0 p i) := fun _ => - (SameC.call hP.bitUnpack _ _).seq ((SameC.call hP.normLt _ _).seq rfl) - exact SameC.seq rfl ((((SameC.call hP.hintUnpack _ _).seq rfl).seq (SameC.ifOk ((SameC.seqR zOne _ _).seq + (SameC.call hP.bitUnpack _).seq ((SameC.call hP.normLt _).seq rfl) + exact SameC.seq rfl ((((SameC.call hP.hintUnpack _).seq rfl).seq (SameC.ifOk ((SameC.seqR zOne _ _).seq (SameC.ifOk (samples.seq compute))))).seq rfl) end diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean index 829f96384..8d87a8c4d 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean @@ -11,6 +11,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Round.UseHint import VerifiedGarbage.Proof.MlDsa.X86_64.Pack.SimpleBitPack import VerifiedGarbage.Proof.MlDsa.X86_64.Pack.Unpack import VerifiedGarbage.Proof.MlDsa.X86_64.Pack.HintUnpack +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Backend /-! # ML-DSA verification on x86-64: the primitives it calls @@ -18,13 +19,15 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Pack.HintUnpack Untrusted: everything here is checked by Lean. The x86-64 implementations of the primitives (`prims`) meet their contracts with at most 16 bytes of stack, never write the stack pointer or load MXCSR, and call at most two -deep (`prims_ok`), so `verify prims p` meets `verifyContract p`. +deep, with any implementation `v` of the polynomial arithmetic +(`prims_okWith`), so `verify (primsWith v.code) p` meets `verifyContract p`. -/ namespace VG.Proof.MlDsa.X86_64.Verify open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Verify open VG.Impl.MlDsa.X86_64 +open VG.Proof.MlDsa.X86_64 (FnOk ArithImpl) /-- The x86-64 implementations of the primitives. -/ def prims : Prims where @@ -42,54 +45,72 @@ def prims : Prims where hintUnpack := Pack.hintBitUnpack normLt := Round.normLt -theorem prims_ok : PrimsOk prims where +/-- The primitives, with the polynomial arithmetic of `B`. -/ +def primsWith (B : Arith.Backend) : Prims := + { prims with + ntt := B.ntt + invNtt := B.invNtt + mul := B.mul + mulAdd := B.mulAdd + sub := B.sub + sfx := B.sfx } + +/-- A function of the polynomial arithmetic satisfies what the proofs of verification need of it. -/ +theorem calleeOf {sig : Sig} {pre : Curry (sig.words X86_64.abi.ptrBits) (Mem → Prop)} + {post : sig.Post X86_64.abi.ptrBits} {wa : Bool} {c : Prog isa} + (h : FnOk (fun S => sig.contract X86_64.abi pre post wa S none) c) : + CalleeOk c (sig.contract X86_64.abi pre post wa 16 none) := + CalleeOk.of_verified h.ver (by decide) h.nosp h.depth h.ctl h.sp + +theorem prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) where ntt := by - have h := Proof.MlDsa.X86_64.Arith.ntt_verified + have h := v.ok.ntt unfold Spec.MlDsa.nttContract Spec.MlDsa.inPlaceContract at h ⊢ - exact CalleeOk.of_verified h (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) - (by lit_decide) (Code.all_of_allInstrs (by lit_decide)) + exact calleeOf h invNtt := by - have h := Proof.MlDsa.X86_64.Arith.nttInv_verified + have h := v.ok.invNtt unfold Spec.MlDsa.nttInvContract Spec.MlDsa.inPlaceContract at h ⊢ - exact CalleeOk.of_verified h (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) - (by lit_decide) (Code.all_of_allInstrs (by lit_decide)) - mul := CalleeOk.of_verified Proof.MlDsa.X86_64.Arith.mul_verified (by decide) - (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - mulAdd := CalleeOk.of_verified Proof.MlDsa.X86_64.Arith.mulAdd_verified (by decide) - (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - sub := CalleeOk.of_verified Proof.MlDsa.X86_64.Arith.sub_verified (by decide) - (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - rejNtt := CalleeOk.of_verified Proof.MlDsa.X86_64.Sample.rejNTT_verified (by decide) + exact calleeOf h + mul := calleeOf v.ok.mul + mulAdd := calleeOf v.ok.mulAdd + sub := calleeOf v.ok.sub + rejNtt := (CalleeOk.of_verified Proof.MlDsa.X86_64.Sample.rejNTT_verified (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - ball := CalleeOk.of_verified Proof.MlDsa.X86_64.Sample.sampleInBall_verified (by decide) + (Code.all_of_allInstrs (by lit_decide)) : + CalleeOk prims.rejNtt _) + ball := (CalleeOk.of_verified Proof.MlDsa.X86_64.Sample.sampleInBall_verified (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - useHint := CalleeOk.of_verified Proof.MlDsa.X86_64.Round.useHint_verified (by decide) + (Code.all_of_allInstrs (by lit_decide)) : + CalleeOk prims.ball _) + useHint := (CalleeOk.of_verified Proof.MlDsa.X86_64.Round.useHint_verified (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - simpleBitPack := CalleeOk.of_verified Proof.MlDsa.X86_64.Pack.simpleBitPack_verified (by decide) + (Code.all_of_allInstrs (by lit_decide)) : + CalleeOk prims.useHint _) + simpleBitPack := (CalleeOk.of_verified Proof.MlDsa.X86_64.Pack.simpleBitPack_verified (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - bitUnpack := CalleeOk.of_verified Proof.MlDsa.X86_64.Pack.bitUnpack_verified (by decide) + (Code.all_of_allInstrs (by lit_decide)) : + CalleeOk prims.simpleBitPack _) + bitUnpack := (CalleeOk.of_verified Proof.MlDsa.X86_64.Pack.bitUnpack_verified (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - unpackT1 := CalleeOk.of_verified Proof.MlDsa.X86_64.Pack.unpackT1_verified (by decide) + (Code.all_of_allInstrs (by lit_decide)) : + CalleeOk prims.bitUnpack _) + unpackT1 := (CalleeOk.of_verified Proof.MlDsa.X86_64.Pack.unpackT1_verified (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - hintUnpack := CalleeOk.of_verified Proof.MlDsa.X86_64.Pack.hintBitUnpack_verified (by decide) + (Code.all_of_allInstrs (by lit_decide)) : + CalleeOk prims.unpackT1 _) + hintUnpack := (CalleeOk.of_verified Proof.MlDsa.X86_64.Pack.hintBitUnpack_verified (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) - normLt := CalleeOk.of_verified Proof.MlDsa.X86_64.Round.normLt_verified (by decide) + (Code.all_of_allInstrs (by lit_decide)) : + CalleeOk prims.hintUnpack _) + normLt := (CalleeOk.of_verified Proof.MlDsa.X86_64.Round.normLt_verified (by decide) (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) + (Code.all_of_allInstrs (by lit_decide)) : + CalleeOk prims.normLt _) -/-- `vg_mldsa*_verify` for the parameter set `p`, calling the x86-64 primitives. -/ -theorem verify_prims {p : Spec.MlDsa.Params} (hp : p ∈ params) : - Verified X86_64.target (verify prims p) (Spec.MlDsa.verifyContract p X86_64.abi 24) := - verify_verified prims_ok hp +/-- `vg_mldsa*_verify` for the parameter set `p`, calling the x86-64 +primitives, with the polynomial arithmetic of `v`. -/ +theorem verify_prims (v : ArithImpl) {p : Spec.MlDsa.Params} (hp : p ∈ params) : + Verified X86_64.target (verify (primsWith v.code) p) (Spec.MlDsa.verifyContract p X86_64.abi 24) := + verify_verified (prims_okWith v) hp end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Sse2.lean b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Sse2.lean new file mode 100644 index 000000000..51dccc5dd --- /dev/null +++ b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Sse2.lean @@ -0,0 +1,16 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Backend + +/-! +# ML-DSA's polynomial arithmetic on x86-64: SSE2 + +A variant of `MlDsaArith` on x86-64 (see `TCB/Emit.lean`): `vg_mldsa_ntt`, +`vg_mldsa_inv_ntt`, `vg_mldsa_multiply_ntt`, `vg_mldsa_multiply_add_ntt`, +`vg_mldsa_add` and `vg_mldsa_sub`, in the baseline ISA (SSE2), which key +generation, signing and verification call. +-/ + +namespace VG.Variants.MlDsaArith.X86_64.Sse2 + +def variant : Proof.MlDsa.X86_64.ArithImpl := .sse2 + +end VG.Variants.MlDsaArith.X86_64.Sse2 From 791054fc47783bf2459661de3968d5ab7d6b5e0e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 17:30:12 +0000 Subject: [PATCH 4/8] ML-DSA on x86-64: polynomial arithmetic in AVX2 Add an AVX2 variant of MlDsaArith on x86-64: vg_mldsa_ntt_avx2, vg_mldsa_inv_ntt_avx2, vg_mldsa_multiply_ntt_avx2, vg_mldsa_multiply_add_ntt_avx2, vg_mldsa_add_avx2 and vg_mldsa_sub_avx2 compute on eight coefficients at a time, four in each 128-bit lane of an AVX2 register. In each lane the VEX.256 form of the SSE2 code does what the SSE2 code does to an xmm register, so the proofs lift the SSE2 lemmas to each lane (ML-KEM's ylanes). The NTT layers with len >= 8 load eight coefficients of each half of a block; len = 4 regroups two blocks with vperm2i128; len = 2 and 1 run the SSE2 gatherings in each lane, with the zetas of each lane arranged by vpshufd/vpblendd or vpermq. Key generation, signing and verification are generic over MlDsaArith, so the emitter generates their _avx2 instances; the Rust API selects them on CPUs with AVX and AVX2 (mldsa_common::Backend). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- README.md | 6 +- docs/algorithms/ml-dsa-44.toml | 2 +- docs/algorithms/ml-dsa-65.toml | 2 +- docs/algorithms/ml-dsa-87.toml | 2 +- .../Artifacts/MlDsaArith/X86_64.lean | 79 +- .../Impl/MlDsa/X86_64/Arith/Avx2.lean | 203 + .../Proof/MlDsa/X86_64/Arith/BackendAvx2.lean | 36 + .../Proof/MlDsa/X86_64/Arith/Mul.lean | 2 +- .../Proof/MlDsa/X86_64/Arith/Mxcsr.lean | 23 +- .../Proof/MlDsa/X86_64/Arith/YAddSub.lean | 220 + .../Proof/MlDsa/X86_64/Arith/YBase.lean | 130 + .../Proof/MlDsa/X86_64/Arith/YLay.lean | 632 ++ .../Proof/MlDsa/X86_64/Arith/YLay21.lean | 526 ++ .../Proof/MlDsa/X86_64/Arith/YMul.lean | 392 + .../Proof/MlDsa/X86_64/Arith/YNtt.lean | 419 + .../Variants/MlDsaArith/X86_64/Avx2.lean | 18 + src/asm/x86_64/mldsa.rs | 1682 ++++ src/asm/x86_64/mldsa44.rs | 3454 +++++++ src/asm/x86_64/mldsa65.rs | 5169 +++++++++++ src/asm/x86_64/mldsa87.rs | 7983 +++++++++++++++++ src/mldsa44.rs | 3 + src/mldsa65.rs | 3 + src/mldsa87.rs | 3 + src/mldsa_common.rs | 62 +- 24 files changed, 21029 insertions(+), 22 deletions(-) create mode 100644 lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YAddSub.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YBase.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay21.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMul.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YNtt.lean create mode 100644 lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean diff --git a/README.md b/README.md index 49aa2153d..e3c64b3bc 100644 --- a/README.md +++ b/README.md @@ -827,7 +827,7 @@ yours to keep: ✅ -✅ SSE2 polynomial arithmetic +✅ AVX2; SSE2 and AVX2 polynomial arithmetic ✅ SHA extensions @@ -843,7 +843,7 @@ yours to keep: ✅ -✅ SSE2 polynomial arithmetic +✅ AVX2; SSE2 and AVX2 polynomial arithmetic ✅ SHA extensions @@ -859,7 +859,7 @@ yours to keep: ✅ -✅ SSE2 polynomial arithmetic +✅ AVX2; SSE2 and AVX2 polynomial arithmetic ✅ SHA extensions diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml index 854e6670e..766d606c3 100644 --- a/docs/algorithms/ml-dsa-44.toml +++ b/docs/algorithms/ml-dsa-44.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa44.rs"] asm = ["mldsa44", "mldsa"] -optimized = { x86_64 = "SSE2 polynomial arithmetic" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" } diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml index e87725611..8835d083b 100644 --- a/docs/algorithms/ml-dsa-65.toml +++ b/docs/algorithms/ml-dsa-65.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa65.rs"] asm = ["mldsa65", "mldsa"] -optimized = { x86_64 = "SSE2 polynomial arithmetic" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" } diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml index 9cfbc9f3a..9a82d4299 100644 --- a/docs/algorithms/ml-dsa-87.toml +++ b/docs/algorithms/ml-dsa-87.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa87.rs"] asm = ["mldsa87", "mldsa"] -optimized = { x86_64 = "SSE2 polynomial arithmetic" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" } diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean index a3d95256b..12caec852 100644 --- a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean +++ b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean @@ -2,6 +2,9 @@ import VerifiedGarbage.TCB.X86_64.Target import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YNtt +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMul +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YAddSub /-! # ML-DSA (FIPS 204) on x86-64: the arithmetic of polynomials @@ -80,6 +83,80 @@ def artifacts : List Artifact := [ code := Impl.MlDsa.X86_64.Arith.sub contract := Spec.MlDsa.subContract X86_64.abi verified := Proof.MlDsa.X86_64.Arith.sub_verified - spSafe := Code.all_of_allInstrs (by lit_decide) }] + spSafe := Code.all_of_allInstrs (by lit_decide) }, + { Spec.MlDsa.nttApi with + name := Spec.MlDsa.nttApi.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.nttApi.doc + (notes := ["The function computes on eight coefficients at a time in AVX2 registers, with a table of \ + the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications \ + (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back \ + before returning."]) + code := Impl.MlDsa.X86_64.Arith.nttAvx2 + contract := Spec.MlDsa.nttContract X86_64.abi + verified := Proof.MlDsa.X86_64.Arith.nttY_verified + spSafe := Code.all_of_allInstrs (by lit_decide) + features := ["avx", "avx2"] + ofSig := ⟨_, _, _, by unfold Spec.MlDsa.nttContract Spec.MlDsa.inPlaceContract; rfl⟩ }, + { Spec.MlDsa.nttInvApi with + name := Spec.MlDsa.nttInvApi.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.nttInvApi.doc + (notes := ["The function computes on eight coefficients at a time in AVX2 registers, with a table of \ + the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications \ + (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back \ + before returning."]) + code := Impl.MlDsa.X86_64.Arith.nttInvAvx2 + contract := Spec.MlDsa.nttInvContract X86_64.abi + verified := Proof.MlDsa.X86_64.Arith.nttInvY_verified + spSafe := Code.all_of_allInstrs (by lit_decide) + features := ["avx", "avx2"] + ofSig := ⟨_, _, _, by unfold Spec.MlDsa.nttInvContract Spec.MlDsa.inPlaceContract; rfl⟩ }, + { Spec.MlDsa.mulApi with + name := Spec.MlDsa.mulApi.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.mulApi.doc + (notes := ["The function computes on eight coefficients at a time in AVX2 registers. It sets MXCSR to \ + `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), \ + through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before \ + returning."]) + code := Impl.MlDsa.X86_64.Arith.mulAvx2 + contract := Spec.MlDsa.mulContract X86_64.abi + verified := Proof.MlDsa.X86_64.Arith.mulY_verified + spSafe := Code.all_of_allInstrs (by lit_decide) + features := ["avx", "avx2"] }, + { Spec.MlDsa.mulAddApi with + name := Spec.MlDsa.mulAddApi.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.mulAddApi.doc + (notes := ["The function computes on eight coefficients at a time in AVX2 registers. It sets MXCSR to \ + `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), \ + through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before \ + returning."]) + code := Impl.MlDsa.X86_64.Arith.mulAddAvx2 + contract := Spec.MlDsa.mulAddContract X86_64.abi + verified := Proof.MlDsa.X86_64.Arith.mulAddY_verified + spSafe := Code.all_of_allInstrs (by lit_decide) + features := ["avx", "avx2"] }, + { Spec.MlDsa.addApi with + name := Spec.MlDsa.addApi.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.addApi.doc + (notes := ["The function computes on eight coefficients at a time in AVX2 registers."]) + code := Impl.MlDsa.X86_64.Arith.addAvx2 + contract := Spec.MlDsa.addContract X86_64.abi + verified := Proof.MlDsa.X86_64.Arith.addY_verified + spSafe := Code.all_of_allInstrs (by lit_decide) + features := ["avx", "avx2"] }, + { Spec.MlDsa.subApi with + name := Spec.MlDsa.subApi.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.subApi.doc + (notes := ["The function computes on eight coefficients at a time in AVX2 registers."]) + code := Impl.MlDsa.X86_64.Arith.subAvx2 + contract := Spec.MlDsa.subContract X86_64.abi + verified := Proof.MlDsa.X86_64.Arith.subY_verified + spSafe := Code.all_of_allInstrs (by lit_decide) + features := ["avx", "avx2"] }] end VG.Artifacts.MlDsaArith.X86_64 diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean new file mode 100644 index 000000000..4c4172ca7 --- /dev/null +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean @@ -0,0 +1,203 @@ +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Backend +import VerifiedGarbage.Impl.MlKem.X86_64.Avx + +/-! +# ML-DSA on x86-64: the polynomial arithmetic with AVX2 + +`vg_mldsa_ntt_avx2`, `vg_mldsa_inv_ntt_avx2`, `vg_mldsa_multiply_ntt_avx2`, +`vg_mldsa_multiply_add_ntt_avx2`, `vg_mldsa_add_avx2` and +`vg_mldsa_sub_avx2` are their SSE2 versions (`Ntt.lean`, `Mul.lean`, +`AddSub.lean`) on eight coefficients at a time, four in each 128-bit lane of +AVX2 registers: in each lane, the VEX.256 form of the SSE2 code (`toY`, +`Impl/MlKem/X86_64/Avx.lean`) does what the SSE2 code does to an `xmm` +register, with `q`, `-q⁻¹ mod 2³²` (and `2⁶⁴ mod q`) in both lanes of +`ymm15`, `ymm14` (and `ymm11`) (`yconsts`). + +The layers of the NTT and its inverse, each a pass over `f` with `rdx` at +the coefficients it loads and `r8` at the zetas of the table: + +* `len ≥ 8` (`ylay`): each block with its zeta in both lanes of `ymm13` + (`yzeta1`), and `len / 8` times the butterflies of the eight + coefficients `w[j]` and the eight `w[j + len]`; +* `len = 4` (`ylay4`): two blocks at a time, their lower halves gathered + into `ymm0` and their upper halves into `ymm1` by `vperm2i128`, with the + zeta of each in a lane of `ymm13` (`yzetaS`: `vpshufd` and `vpblendd`); +* `len = 2` and `len = 1` (`ylay2`, `ylay1`): in each lane `l`, what + `vlay2` and `vlay1` do to `xmm0` and `xmm1` (or `xmm2`), with lane `l` of + the two loads of 32 bytes holding the coefficients at `16l` and + `32 + 16l`, and the zetas of the blocks there in lane `l` of `ymm13` + (`yzetaS`, or `vpermq` of eight zetas). + +`NTT⁻¹` then scales every coefficient (`yscale`). The multiplications run +inside `withMxcsr` as in the SSE2 code: for `vg_mldsa_multiply*_ntt_avx2`, +through the last 8 bytes of `h`, whose last eight coefficients are loaded +to `ymm6` first; the loop stores the first 248, and the last eight are +stored after MXCSR is loaded back. Every function clears the upper halves of +the vector registers before returning (`vzeroupper`). Every address and +branch depends only on the pointers. +-/ + +namespace VG.Impl.MlDsa.X86_64.Arith + +open VG.X86_64 +open VG.Impl.MlKem.X86_64 (xb xmov withMxcsr rcxLoop toY yconst) + +/-- `q` in the doublewords of `ymm15` and `-q⁻¹ mod 2³²` in those of `ymm14`. -/ +def yconsts : List Instr := yconst .xmm15 8380417 ++ yconst .xmm14 4236238847 + +/-- `vzeroupper`. -/ +def yepi : List Instr := [.vop .vzeroupper] + +/-! ## The NTT and its inverse -/ + +/-- The zeta at `[r8]` in every doubleword of both lanes of `ymm13`, and in +those of `ymm12`: `vzeta 0` in each lane. -/ +def yzeta1 : List Instr := + .vbroadcasti128 .xmm13 (at_ .r8 0) :: toY [.xop (.pshufd .xmm13 .xmm13 0), .xop (.pshufd .xmm12 .xmm13 0xF5)] + +/-- The four zetas at `[r8]` in both lanes, arranged by `vpshufd` with `o₀` +into lane 0 of `ymm13` and with `o₁` into lane 1 (through `ymm2`), and its +odd doublewords in the even ones of `ymm12`. -/ +def yzetaS (o₀ o₁ : BitVec 8) : List Instr := + .vbroadcasti128 .xmm13 (at_ .r8 0) :: + toY [.xop (.pshufd .xmm2 .xmm13 o₁), .xop (.pshufd .xmm13 .xmm13 o₀)] ++ + [.vop (.vpblendd .l256 .xmm13 .xmm13 .xmm2 0xF0)] ++ toY [.xop (.pshufd .xmm12 .xmm13 0xF5)] + +/-- A layer with `len ≥ 8` and butterflies `bf`: its `128 / len` blocks, the +first with the zeta `k`, the zeta pointer moving by `dz` bytes. -/ +def ylay (bf : List Instr) (len k : Nat) (dz : BitVec 32) : Prog isa := + .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k) ++ + [.mov32 .rax (.imm (BitVec.ofNat 32 (128 / len)))])) <| + .loop (.seq (.block (yzeta1 ++ [.alu .add .r8 (.imm dz)])) + (.seq (rcxLoop (len / 8) ([.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), + .vmovdquLoad .l256 .xmm1 (at_ .rdx (4 * len))] ++ toY bf ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx (4 * len)) .xmm3, + .alu .add .rdx (.imm 32)])) + (.block [.alu .add .rdx (.imm (BitVec.ofNat 32 (4 * len))), .alu .sub .rax (.imm 1)]))) .ne + +/-- The layer with `len = 4`, two blocks at a time: the first with the zeta +`k`, the zetas of a pair at `[r8]` arranged by `yzetaS o₀ o₁`, the zeta +pointer moving by `dz` bytes. -/ +def ylay4 (bf : List Instr) (k : Nat) (o₀ o₁ : BitVec 8) (dz : BitVec 32) : Prog isa := + .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k))) <| + rcxLoop 16 ([.vmovdquLoad .l256 .xmm4 (at_ .rdx 0), .vmovdquLoad .l256 .xmm5 (at_ .rdx 32)] ++ + yzetaS o₀ o₁ ++ + [.alu .add .r8 (.imm dz), .vop (.vperm2i128 .xmm0 .xmm4 .xmm5 0x20), + .vop (.vperm2i128 .xmm1 .xmm4 .xmm5 0x31)] ++ toY bf ++ + [.vop (.vperm2i128 .xmm4 .xmm0 .xmm3 0x20), .vop (.vperm2i128 .xmm5 .xmm0 .xmm3 0x31), + .vmovdquStore .l256 (at_ .rdx 0) .xmm4, .vmovdquStore .l256 (at_ .rdx 32) .xmm5, + .alu .add .rdx (.imm 64)]) + +/-- `vlay2`'s gathering of the halves of two blocks (`Ntt.lean`). -/ +def gath2 : List Instr := [xmov .xmm2 .xmm0, xb .punpcklqdq .xmm0 .xmm1, xb .punpckhqdq .xmm2 .xmm1, xmov .xmm1 .xmm2] + +/-- `vlay2`'s interleaving back. -/ +def scat2 : List Instr := [xmov .xmm1 .xmm0, xb .punpcklqdq .xmm0 .xmm3, xb .punpckhqdq .xmm1 .xmm3] + +/-- The layer with `len = 2`, four blocks at a time: in each lane, `vlay2`'s +two (`gath2`, `bf`, `scat2`), with their zetas from `[r8]` arranged by +`yzetaS o₀ o₁`, the zeta pointer moving by `dz` bytes. -/ +def ylay2 (bf : List Instr) (k : Nat) (o₀ o₁ : BitVec 8) (dz : BitVec 32) : Prog isa := + .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k))) <| + rcxLoop 16 ([.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm1 (at_ .rdx 32)] ++ + yzetaS o₀ o₁ ++ [.alu .add .r8 (.imm dz)] ++ toY (gath2 ++ bf ++ scat2) ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1, .alu .add .rdx (.imm 64)]) + +/-- `vlay1`'s gathering of the pairs of four blocks (`Ntt.lean`). -/ +def gath1 : List Instr := + [.xop (.pshufd .xmm0 .xmm0 0xD8), .xop (.pshufd .xmm2 .xmm2 0xD8), xmov .xmm1 .xmm0, + xb .punpcklqdq .xmm0 .xmm2, xb .punpckhqdq .xmm1 .xmm2] + +/-- `vlay1`'s interleaving back. -/ +def scat1 : List Instr := [xmov .xmm1 .xmm0, xb .punpckldq .xmm0 .xmm3, xb .punpckhdq .xmm1 .xmm3] + +/-- The eight zetas at `[r8]` in the order the lanes of `ylay1` take them +for `NTT`: the first two and the fifth and sixth in lane 0, the others in +lane 1 (`vpermq`). -/ +def yzeta8 : List Instr := + [.vmovdquLoad .l256 .xmm13 (at_ .r8 0), .vop (.vpermq .xmm13 .xmm13 0xD8)] ++ + toY [.xop (.pshufd .xmm12 .xmm13 0xF5)] + +/-- The same for `NTT⁻¹`, whose blocks take the zetas in decreasing order: +the eighth, seventh, fourth and third in lane 0, the others in lane 1. -/ +def yzeta8R : List Instr := + [.vmovdquLoad .l256 .xmm13 (at_ .r8 0), .vop (.vpermq .xmm13 .xmm13 0x27)] ++ + toY [.xop (.pshufd .xmm13 .xmm13 0xB1), .xop (.pshufd .xmm12 .xmm13 0xF5)] + +/-- The layer with `len = 1`, eight blocks at a time: in each lane, +`vlay1`'s four, with their zetas from `[r8]` (`zl`), the zeta pointer moving +by `dz` bytes. -/ +def ylay1 (bf : List Instr) (k : Nat) (zl : List Instr) (dz : BitVec 32) : Prog isa := + .seq (.block ([.mov .rdx (.reg .rdi)] ++ leaR .r8 .rsi (4 * k))) <| + rcxLoop 16 ([.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm2 (at_ .rdx 32)] ++ zl ++ + [.alu .add .r8 (.imm dz)] ++ toY (gath1 ++ bf ++ scat1) ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1, .alu .add .rdx (.imm 64)]) + +/-- Every coefficient times `256⁻¹ mod q`: `vscale` in each lane. -/ +def yscale : Prog isa := + .seq (.block ([.mov .rdx (.reg .rdi)] ++ yconst .xmm13 16382 ++ [.vop (.vmovdqa .l256 .xmm12 .xmm13)])) + (rcxLoop 32 ([.vmovdquLoad .l256 .xmm3 (at_ .rdx 0)] ++ toY (vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ + vcsub .xmm3 .xmm2) ++ [.vmovdquStore .l256 (at_ .rdx 0) .xmm3, .alu .add .rdx (.imm 32)])) + +/-- The table and the constants. -/ +def ypro : List Instr := dwordTab zmTab 256 .rsi ++ yconsts + +def nttAvx2 : Prog isa := withMxcsr .rsi 768 <| + .seq (.block ypro) (.seq (ylay vbfly 128 1 4) (.seq (ylay vbfly 64 2 4) (.seq (ylay vbfly 32 4 4) + (.seq (ylay vbfly 16 8 4) (.seq (ylay vbfly 8 16 4) (.seq (ylay4 vbfly 32 0x00 0x55 8) + (.seq (ylay2 vbfly 64 0xA0 0xF5 16) (.seq (ylay1 vbfly 128 yzeta8 32) (.block yepi))))))))) + +def nttInvAvx2 : Prog isa := withMxcsr .rsi 768 <| + .seq (.block ypro) (.seq (ylay1 vibfly 248 yzeta8R (-32)) (.seq (ylay2 vibfly 124 0x5F 0x0A (-16)) + (.seq (ylay4 vibfly 62 0x55 0x00 (-8)) (.seq (ylay vibfly 8 31 (-4)) (.seq (ylay vibfly 16 15 (-4)) + (.seq (ylay vibfly 32 7 (-4)) (.seq (ylay vibfly 64 3 (-4)) (.seq (ylay vibfly 128 1 (-4)) + (.seq yscale (.block yepi)))))))))) + +/-! ## Products -/ + +/-- The constants and `2⁶⁴ mod q` in the doublewords of `ymm11`. -/ +def ymulPro : List Instr := yconsts ++ yconst .xmm11 2365951 + +/-- The coefficients of `f`, `g` and `h` to `ymm3`, `ymm13` and `ymm5`. -/ +def ymulLoads : List Instr := + [.vmovdquLoad .l256 .xmm3 (at_ .rsi 0), .vmovdquLoad .l256 .xmm13 (at_ .rdx 0), + .vmovdquLoad .l256 .xmm5 (at_ .rdi 0)] + +/-- Store `ymm3` to `h` and advance the three pointers. -/ +def ymulTail : List Instr := + [.vmovdquStore .l256 (at_ .rdi 0) .xmm3, .alu .add .rdi (.imm 32), .alu .add .rsi (.imm 32), + .alu .add .rdx (.imm 32)] + +/-- The last eight coefficients, with those of `h` in `ymm6`, to `ymm3`. -/ +def ymulLast (core : List Instr) : List Instr := + [.vmovdquLoad .l256 .xmm3 (at_ .rsi 0), .vmovdquLoad .l256 .xmm13 (at_ .rdx 0), + .vop (.vmovdqa .l256 .xmm5 .xmm6)] ++ toY core + +/-- `mulFn` on eight coefficients at a time. -/ +def ymulFn (core : List Instr) : Prog isa := + .seq (.block [.mov .r8 (.reg .rdi), .vmovdquLoad .l256 .xmm6 (at_ .rdi 992)]) + (.seq (withMxcsr .r8 1016 + (.seq (.block ymulPro) (.seq (rcxLoop 31 (ymulLoads ++ toY core ++ ymulTail)) (.block (ymulLast core))))) + (.block ([.vmovdquStore .l256 (at_ .rdi 0) .xmm3] ++ yepi))) + +def mulAvx2 : Prog isa := ymulFn mulCore + +def mulAddAvx2 : Prog isa := ymulFn mulAddCore + +/-! ## Sums and differences -/ + +/-- The body of `add` and `sub` on eight coefficients at a time. -/ +def yaccBody (op : XBinOp) (fix : List Instr) : List Instr := + [.vmovdquLoad .l256 .xmm0 (at_ .rdi 0), .vmovdquLoad .l256 .xmm1 (at_ .rsi 0)] ++ toY (xb op .xmm0 .xmm1 :: fix) ++ + [.vmovdquStore .l256 (at_ .rdi 0) .xmm0, .alu .add .rdi (.imm 32), .alu .add .rsi (.imm 32)] + +def addAvx2 : Prog isa := + .seq (.block (yconst .xmm15 8380417)) (.seq (rcxLoop 32 (yaccBody .paddd (vcsub .xmm0 .xmm2))) (.block yepi)) + +def subAvx2 : Prog isa := + .seq (.block (yconst .xmm15 8380417)) (.seq (rcxLoop 32 (yaccBody .psubd (vcadd .xmm0 .xmm2))) (.block yepi)) + +/-- The AVX2 code. -/ +def Backend.avx2 : Backend := ⟨nttAvx2, nttInvAvx2, mulAvx2, mulAddAvx2, addAvx2, subAvx2, "_avx2"⟩ + +end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean new file mode 100644 index 000000000..157073d12 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean @@ -0,0 +1,36 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Backend +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YNtt +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMul +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YAddSub + +/-! +# ML-DSA on x86-64: the polynomial arithmetic with AVX2, as an `ArithImpl` + +Untrusted: everything here is checked by Lean. The AVX2 code +(`Impl/MlDsa/X86_64/Arith/Avx2.lean`) meets what the callers of the +polynomial arithmetic need of it (`FnOk`), and requires AVX and AVX2. +-/ + +namespace VG.Proof.MlDsa.X86_64 + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith + +/-- The AVX2 code. -/ +def ArithImpl.avx2 : ArithImpl where + code := .avx2 + ok := + { ntt := FnOk.of Arith.nttY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + invNtt := FnOk.of Arith.nttInvY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + mul := FnOk.of Arith.mulY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + mulAdd := FnOk.of Arith.mulAddY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + add := FnOk.of Arith.addY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + sub := FnOk.of Arith.subY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) } + features := ["avx", "avx2"] + +end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean index 7cdf5d3b0..dcff58350 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mul.lean @@ -286,7 +286,7 @@ theorem fn_ok {t : Poly → Poly → Poly → Poly} {hPre : Mem → Addr → Pro s3.gpr .rdi = coeffAddr h 252 ∧ Frame [pR h] σ.mem s3.mem ∧ (∀ k < 252, (coeffAt s3.mem h k).toNat = (R[k]!).val) ∧ ∀ e < 4, (dword (s3.xmm .xmm3) e).toNat = (R[252 + e]!).val) - fun s2 k2 f2 x2 => ?_) fun s4 ⟨s3, ⟨kk, hdi, fr, dn, ln⟩, f4, k4, x4⟩ => ?_) + fun s2 k2 f2 x2 _ => ?_) fun s4 ⟨s3, ⟨kk, hdi, fr, dn, ln⟩, f4, k4, x4, _⟩ => ?_) · have fσ2 : Frame [mxH h] σ.mem s2.mem := by rw [← m1]; exact f2 refine WP.mono (WP.keep [.rax, .rdi, .rsi, .rdx, .rcx] (WP.seq (WP.mono (mulPro_ok s2) fun w ⟨cw, xw, x6, kw, mw⟩ => ?_) (Q := fun (s3 : State) => s3.gpr .rdi = coeffAddr h 252 ∧ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean index d87f9e044..ef50cbe5b 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Mxcsr.lean @@ -29,34 +29,37 @@ those bytes, and nothing more than they and MXCSR change after it. -/ theorem withMxcsrH_ok {c : Prog isa} {r : Reg} (hr : r ≠ .r11 ∧ r ≠ .rax) (rs : List Reg) (hrs : r ∉ rs ∧ Reg.r11 ∉ rs) {p : Addr} {s : State} {Q : State → Prop} (hsi : s.gpr r = p) (hw : pR p ∈ s.wr) (hk : writesOnly rs c = true) - (hc : ∀ s1, Keep [.rax, .r11] s s1 → Frame [mxH p] s.mem s1.mem → s1.xmm = s.xmm → WP isa c s1 Q) : + (hc : ∀ s1, Keep [.rax, .r11] s s1 → Frame [mxH p] s.mem s1.mem → s1.xmm = s.xmm → s1.ymmHi = s.ymmHi → + WP isa c s1 Q) : WP isa (withMxcsr r 1016 c) s fun s' => - ∃ s2, Q s2 ∧ Frame [mxH p] s2.mem s'.mem ∧ Keep [] s2 s' ∧ s'.xmm = s2.xmm := by + ∃ s2, Q s2 ∧ Frame [mxH p] s2.mem s'.mem ∧ Keep [] s2 s' ∧ s'.xmm = s2.xmm ∧ s'.ymmHi = s2.ymmHi := by have h0 := mxH_in hw 1016 (by decide) have h0' := mxH_in (List.mem_append_right s.rd hw) 1016 (by decide) have h4 := mxH_in hw 1020 (by decide) simp only [withMxcsr] refine WP.seq (WP.mono (Q := fun (s1 : State) => s1.gpr .r11 = (s.mxcsr &&& 0xFFFF).setWidth 64 ∧ Keep [.r11] s s1 ∧ - Frame [mxH p] s.mem s1.mem ∧ s1.xmm = s.xmm) (by + Frame [mxH p] s.mem s1.mem ∧ s1.xmm = s.xmm ∧ s1.ymmHi = s.ymmHi) (by vrunm [hsi, h0, h0', Mem.readW_writeW_self32, hr.1] refine ⟨by rw [BitVec.setWidth_setWidth_of_le _ (by decide), BitVec.setWidth_eq], ⟨fun r hr => ?_, rfl, rfl⟩, (Frame.refl _ _).writeW (List.mem_singleton_self _) _ - (Offset.contains p (by decide) (by decide) (by decide))⟩ + (Offset.contains p (by decide) (by decide) (by decide)), + by simp only [RegUpd.ymmHi_setReg, RegUpd.ymmHi_setFlags]⟩ simp only [List.mem_singleton] at hr - simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false]) fun s1 ⟨h11, k1, f1, x1⟩ => ?_) + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false]) fun s1 ⟨h11, k1, f1, x1, y1⟩ => ?_) have hsi1 : s1.gpr r = p := by rw [k1.gpr (by simpa using hr.1), hsi] have h4' : InRegions s1.wr (p + BitVec.ofNat 64 (1016 + 4)) 4 := by rw [k1.2.2]; exact h4 have h4'' : InRegions (s1.rd ++ s1.wr) (p + BitVec.ofNat 64 (1016 + 4)) 4 := let ⟨r, hr, hc⟩ := h4'; ⟨r, List.mem_append_right _ hr, hc⟩ refine WP.seq (WP.seq (WP.mono (Q := fun (s2 : State) => Keep [.rax] s1 s2 ∧ Frame [mxH p] s1.mem s2.mem ∧ - s2.xmm = s1.xmm) + s2.xmm = s1.xmm ∧ s2.ymmHi = s1.ymmHi) (by vrunm [hsi1, h4', h4'', Mem.readW_writeW_self32, hr.2] refine ⟨⟨fun r hr => ?_, rfl, rfl⟩, (Frame.refl _ _).writeW (List.mem_singleton_self _) _ - (Offset.contains p (by decide) (by decide) (by decide))⟩ + (Offset.contains p (by decide) (by decide) (by decide)), by simp only [RegUpd.ymmHi_setReg]⟩ simp only [List.mem_singleton] at hr - simp only [RegUpd.gpr_setReg, hr, ite_false]) fun s2 ⟨k2, f2, x2⟩ => ?_)) - refine WP.seq (WP.mono (WP.keep _ (hc s2 ((k1.trans k2).mono (by simp)) (f1.trans f2) (x2.trans x1)) hk) + simp only [RegUpd.gpr_setReg, hr, ite_false]) fun s2 ⟨k2, f2, x2, y2⟩ => ?_)) + refine WP.seq (WP.mono (WP.keep _ (hc s2 ((k1.trans k2).mono (by simp)) (f1.trans f2) (x2.trans x1) + (y2.trans y1)) hk) fun s3 ⟨hq, k3⟩ => ?_) have k23 := k2.trans k3 have hsi3 : s3.gpr r = p := by rw [k23.gpr (by simp [hr.2, hrs.1]), hsi1] @@ -68,6 +71,6 @@ theorem withMxcsrH_ok {c : Prog isa} {r : Reg} (hr : r ≠ .r11 ∧ r ≠ .rax) subst h4 vrunm [hsi3, h113, h03, h03', Mem.readW_writeW_self32, ldmxcsr_ok] exact ⟨_, hq, (Frame.refl _ _).writeW (List.mem_singleton_self _) _ - (Offset.contains p (by decide) (by decide) (by decide)), ⟨fun _ _ => rfl, rfl, rfl⟩, rfl⟩ + (Offset.contains p (by decide) (by decide) (by decide)), ⟨fun _ _ => rfl, rfl, rfl⟩, rfl, rfl⟩ end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YAddSub.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YAddSub.lean new file mode 100644 index 000000000..701332da3 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YAddSub.lean @@ -0,0 +1,220 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YBase +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub + +/-! +# ML-DSA on x86-64: `vg_mldsa_add_avx2` and `vg_mldsa_sub_avx2` + +Untrusted: everything here is checked by Lean. Each iteration of the loop +loads eight coefficients of `f` and of `g` and, in each lane, does what an +iteration of `vg_mldsa_add` (`vg_mldsa_sub`) does (`AddSub.lean`), whose +proof holds of each lane (`ylanes`), and stores the eight results to `f` +(`YAddSub.step`); the loop leaves `f` with all 256 (`YAddSub.fn_ok`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes yld_ok yconst_ok WP.keep writesOnly gprPreserved_of ifp ifn + ptr_step GOnly wp_rcxLoopY add_ofNat_zero lane_setReg lane_setFlags sx32 State.setMem_ymm) +open VG.Impl.MlKem.X86_64 (xb xmov toY yconst) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs) + +theorem addFixX_ok (s : State) (hq : s.xmm .xmm15 = qV) : + WP isa (.block (xb .paddd .xmm0 .xmm1 :: vcsub .xmm0 .xmm2)) s fun s' => + s'.xmm .xmm0 = addV (s.xmm .xmm0) (s.xmm .xmm1) ∧ XOnly [.xmm0, .xmm2] s s' := by + simp only [vcsub, vcadd, xmov, xb] + vrun [eval_movdqa] + rw [hq] + exact ⟨rfl, by xonly⟩ + +theorem subFixX_ok (s : State) (hq : s.xmm .xmm15 = qV) : + WP isa (.block (xb .psubd .xmm0 .xmm1 :: vcadd .xmm0 .xmm2)) s fun s' => + s'.xmm .xmm0 = subV (s.xmm .xmm0) (s.xmm .xmm1) ∧ XOnly [.xmm0, .xmm2] s s' := by + simp only [vcadd, xmov, xb] + vrun [eval_movdqa] + rw [hq] + exact ⟨rfl, by xonly⟩ + +theorem lane_addFix : laneSseBlock (toY (xb .paddd .xmm0 .xmm1 :: vcsub .xmm0 .xmm2)) = + some (xb .paddd .xmm0 .xmm1 :: vcsub .xmm0 .xmm2) := by decide +kernel + +theorem lane_subFix : laneSseBlock (toY (xb .psubd .xmm0 .xmm1 :: vcadd .xmm0 .xmm2)) = + some (xb .psubd .xmm0 .xmm1 :: vcadd .xmm0 .xmm2) := by decide +kernel + +namespace YAddSub + +/-- After `i` vectors of eight, each coefficient before `8i` is `v k`. -/ +structure Inv (s₀ : State) (v : Nat → BitVec 32) (i : Nat) (s : State) : Prop where + rdi : s.gpr .rdi = s₀.gpr .rdi + BitVec.ofNat 64 (32 * i) + rsi : s.gpr .rsi = s₀.gpr .rsi + BitVec.ofNat 64 (32 * i) + rd : s.rd = s₀.rd + wr : s.wr = s₀.wr + q : ∀ l < 2, s.lane .xmm15 l = qV + frame : Frame [pR (s₀.gpr .rdi)] s₀.mem s.mem + coeff : ∀ k < 256, coeffAt s.mem (s₀.gpr .rdi) k = if k < 8 * i then v k else coeffAt s₀.mem (s₀.gpr .rdi) k + +section +variable {t : Poly → Poly → Poly} {s₀ : State} (hp : (accK t).pre s₀) + {op : XBinOp} {fix : List Instr} {F : BitVec 128 → BitVec 128 → BitVec 128} + {L : BitVec 32 → BitVec 32 → BitVec 32} + (hF : ∀ (s : State), s.xmm .xmm15 = qV → + WP isa (.block (xb op .xmm0 .xmm1 :: fix)) s fun s' => + s'.xmm .xmm0 = F (s.xmm .xmm0) (s.xmm .xmm1) ∧ XOnly [.xmm0, .xmm2] s s') + (hY : laneSseBlock (toY (xb op .xmm0 .xmm1 :: fix)) = some (xb op .xmm0 .xmm1 :: fix)) + (hL : ∀ x y : BitVec 128, ∀ e < 4, dword (F x y) e = L (dword x e) (dword y e)) +include hp hF hY hL + +/-- An iteration, which stores `F` of the vectors of `f` and `g` in each lane. -/ +theorem step {i : Nat} (hi : i < 32) {s : State} + (hI : Inv s₀ (fun k => L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) i s) : + WP isa (.block (yaccBody op fix ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) s fun s' => + Inv s₀ (fun k => L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) (i + 1) s' ∧ + s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by + have j0 : 8 * i + 8 ≤ 256 := by omega + have hw : pR (s₀.gpr .rdi) ∈ s.wr := by rw [hI.wr, hp.2.1]; simp + have hr : pR (s₀.gpr .rsi) ∈ s.rd ++ s.wr := by rw [hI.rd, hI.wr, hp.1]; simp + have e1 : s.gpr .rdi + BitVec.ofNat 64 0 = coeffAddr (s₀.gpr .rdi) (8 * i) := by + rw [add_ofNat_zero, hI.rdi]; congr 2; omega + have e2 : s.gpr .rsi + BitVec.ofNat 64 0 = coeffAddr (s₀.gpr .rsi) (8 * i) := by + rw [add_ofNat_zero, hI.rsi]; congr 2; omega + rw [yaccBody, List.append_assoc, List.append_assoc, WP.block_append_iff, + show ∀ a b : Instr, [a, b] = [a] ++ [b] from fun _ _ => rfl, WP.block_append_iff] + refine WP.mono (yld_ok (by rw [e1]; exact f_in32 (List.mem_append_right _ hw) j0)) fun s1 ⟨L1, o1⟩ => ?_ + refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr, e2]; exact ⟨_, hr, pR_contains32 _ j0⟩)) + fun s2 ⟨L2, o2⟩ => ?_ + rw [WP.block_append_iff] + have o12 := o1.trans o2 + refine WP.mono (ylanes hY (P := fun l t => t.xmm .xmm0 = F ((s2.proj l).xmm .xmm0) ((s2.proj l).xmm .xmm1)) + fun l hl => hF _ (by rw [State.proj_xmm, o12.lane _ (by decide) l hl]; exact hI.q l hl)) + fun s3 ⟨B3, o3⟩ => ?_ + have o13 := o12.trans o3 + have g3 : s3.gpr .rdi = coeffAddr (s₀.gpr .rdi) (8 * i) := by rw [o13.gpr, ← e1, add_ofNat_zero] + have g3' : s3.gpr .rsi = coeffAddr (s₀.gpr .rsi) (8 * i) := by rw [o13.gpr, ← e2, add_ofNat_zero] + have w0 : InRegions s3.wr (s3.gpr .rdi) 32 := by rw [o13.wr, g3]; exact f_in32 hw j0 + vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, + State.setMem_ymm, w0, sx32] + refine ⟨⟨?_, ?_, ?_, ?_, fun l hl => ?_, ?_, fun k hk => ?_⟩, ?_⟩ + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr] + rw [o13.gpr, hI.rdi]; exact ptr_step _ i 32 + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr] + rw [o13.gpr, hI.rsi]; exact ptr_step _ i 32 + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; rw [o13.rd, hI.rd] + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; rw [o13.wr, hI.wr] + · simp only [lane_setReg, lane_setFlags, State.setMem_lane]; rw [o13.lane _ (by decide) l hl]; exact hI.q l hl + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem] + rw [g3, o13.mem]; exact hI.frame.writeW (List.mem_singleton_self _) _ (pR_contains32 _ j0) + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem] + rw [g3, o13.mem, coeffAt_write256 _ _ j0 _ hk] + split + · rename_i h + rw [ifp (show k < 8 * (i + 1) by omega), State.ymm, extract_ymm _ _ (by omega)] + have hc : ∀ l < 2, ∀ e < 4, dword (s3.lane .xmm0 l) e = + L (coeffAt s₀.mem (s₀.gpr .rdi) (8 * i + 4 * l + e)) (coeffAt s₀.mem (s₀.gpr .rsi) (8 * i + 4 * l + e)) := + fun l hl e he => by + rw [← State.proj_xmm, B3 l hl, hL _ _ _ he, State.proj_xmm, State.proj_xmm, + o2.lane _ (by decide) l hl, L1 l hl, L2 l hl, o1.gpr, o1.mem, e1, e2, dword_readW _ _ he, + dword_readW _ _ he, lane_load, lane_load, coeffAddr_add, coeffAddr_add, ← coeffAt_eq, ← coeffAt_eq, + hI.coeff _ (by omega), ifn (by omega), + coeffAt_frame hI.frame (by simpa using hp.2.2.1.symm) (by rw [n_eq]; omega)] + split + · rename_i h4 + have := hc 0 (by decide) (k - 8 * i) h4 + rw [show 8 * i + 4 * 0 + (k - 8 * i) = k by omega] at this + exact this + · rename_i h4 + have := hc 1 (by decide) (k - 8 * i - 4) (by omega) + rw [show 8 * i + 4 * 1 + (k - 8 * i - 4) = k by omega] at this + exact this + · rename_i h + rw [hI.coeff k hk] + by_cases h' : k < 8 * i + · rw [ifp h', ifp (by omega)] + · rw [ifn h', ifn (by omega)] + · exact ⟨by rw [o13.gpr], by rw [o13.gpr]⟩ + +/-- The whole function, from its precondition. -/ +theorem fn_ok (hv : ∀ k < 256, (L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)).toNat = + ((t (polyAt s₀.mem (s₀.gpr .rdi)) (polyAt s₀.mem (s₀.gpr .rsi)))[k]!).val) + (hc : writesOnly [.rax, .rdi, .rsi, .rcx] (.seq (.block (yconst .xmm15 8380417)) + (.seq (VG.Impl.MlKem.X86_64.rcxLoop 32 (yaccBody op fix)) (.block yepi))) = true) + (hm : Code.allInstrs (fun i => !loadsMxcsr i) (.seq (.block (yconst .xmm15 8380417)) + (.seq (VG.Impl.MlKem.X86_64.rcxLoop 32 (yaccBody op fix)) (.block yepi)) : Prog isa) = true) : + ∃ tr s', Exec isa (.seq (.block (yconst .xmm15 8380417)) + (.seq (VG.Impl.MlKem.X86_64.rcxLoop 32 (yaccBody op fix)) (.block yepi))) s₀ tr s' ∧ + abiPreserved s₀ s' ∧ (accK t).post s₀ s' := by + have hW : WP isa (.seq (.block (yconst .xmm15 8380417)) + (.seq (VG.Impl.MlKem.X86_64.rcxLoop 32 (yaccBody op fix)) (.block yepi))) s₀ fun s' => + Frame [pR (s₀.gpr .rdi)] s₀.mem s'.mem ∧ ∀ k < 256, coeffAt s'.mem (s₀.gpr .rdi) k = + L (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k) := by + refine WP.seq (WP.mono (yconst_ok .xmm15 _ s₀) fun w ⟨lq, k1, m1, _, _⟩ => ?_) + refine WP.seq (WP.mono (wp_rcxLoopY (N := 32) (by decide) (by decide) _ (fun u o hy _ => + ⟨by rw [o.keep.gpr (by decide), k1.gpr (by decide), Nat.mul_zero, add_ofNat_zero], + by rw [o.keep.gpr (by decide), k1.gpr (by decide), Nat.mul_zero, add_ofNat_zero], + by rw [o.keep.2.1, k1.2.1], by rw [o.keep.2.2, k1.2.2], + fun l hl => by + rw [show u.lane .xmm15 l = w.lane .xmm15 l by simp only [State.lane]; rw [o.xmm, hy], lq l hl]; decide, + by rw [o.mem, m1]; exact Frame.refl _ _, fun k _ => by rw [o.mem, m1, ifn (by omega)]⟩) + fun i hi u hI => step hp hF hY hL hi hI) fun u hI => ?_) + refine WP.mono (Q := fun (u' : State) => u'.mem = u.mem) (by simp only [yepi]; vrund; rfl) fun u' hm' => ?_ + rw [hm'] + exact ⟨hI.frame, fun k hk => by rw [hI.coeff k hk, ifp (by omega)]⟩ + obtain ⟨tr, s', he, ⟨hf, hco⟩, hk⟩ := WP.keep _ hW hc + refine ⟨tr, s', he, abiPreserved_of_exec hm he (gprPreserved_of hk (by decide) hf ?_), + polyIs_of_toNat fun k hk => ?_⟩ + · simpa using hp.2.2.2.1 + · rw [n_eq] at hk + rw [hco k hk] + exact hv k hk + +end + +end YAddSub + +end VG.Proof.MlDsa.X86_64.Arith + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Spec.MlDsa (n) + +theorem addY_correct (s : State) (hs : (accK Spec.MlDsa.add).pre s) : + ∃ t s', Exec isa addAvx2 s t s' ∧ abiPreserved s s' ∧ (accK Spec.MlDsa.add).post s s' := + YAddSub.fn_ok hs (op := .paddd) (fix := vcsub .xmm0 .xmm2) (L := fun a b => csubL (a + b)) addFixX_ok + lane_addFix (fun x y e he => dword_addV x y he) + (fun k hk => by + have hk' : k < n := by rw [n_eq]; exact hk + rw [add_get _ _ hk', addD_toNat (by rw [← polyAt_val hs.2.2.2.2.2.1 hk']; exact val_lt _) + (by rw [← polyAt_val hs.2.2.2.2.2.2 hk']; exact val_lt _), + ← polyAt_val hs.2.2.2.2.2.1 hk', ← polyAt_val hs.2.2.2.2.2.2 hk', val_add]) + (by decide +kernel) (by decide +kernel) + +theorem subY_correct (s : State) (hs : (accK Spec.MlDsa.sub).pre s) : + ∃ t s', Exec isa subAvx2 s t s' ∧ abiPreserved s s' ∧ (accK Spec.MlDsa.sub).post s s' := + YAddSub.fn_ok hs (op := .psubd) (fix := vcadd .xmm0 .xmm2) (L := fun a b => caddL (a - b)) subFixX_ok + lane_subFix (fun x y e he => dword_subV x y he) + (fun k hk => by + have hk' : k < n := by rw [n_eq]; exact hk + rw [sub_get _ _ hk', subD_toNat (by rw [← polyAt_val hs.2.2.2.2.2.1 hk']; exact val_lt _) + (by rw [← polyAt_val hs.2.2.2.2.2.2 hk']; exact val_lt _), + ← polyAt_val hs.2.2.2.2.2.1 hk', ← polyAt_val hs.2.2.2.2.2.2 hk', val_sub]) + (by decide +kernel) (by decide +kernel) + +theorem addY_ct : ConstantTime isa (accK Spec.MlDsa.add).pre (accK Spec.MlDsa.add).pub addAvx2 := + VG.Taint.constantTime (A := taint) accτ acc_agree (by taint_decide) + +theorem subY_ct : ConstantTime isa (accK Spec.MlDsa.sub).pre (accK Spec.MlDsa.sub).pub subAvx2 := + VG.Taint.constantTime (A := taint) accτ acc_agree (by taint_decide) + +theorem addY_verified : Verified X86_64.target addAvx2 (Spec.MlDsa.addContract X86_64.abi) := + Verified.of_correct addY_correct addY_ct (by + mldsa_implies [Spec.MlDsa.addContract, Spec.MlDsa.accSig, accK, X86_64.abi, X86_64.argRegs] + [accSat] using accSat) + +theorem subY_verified : Verified X86_64.target subAvx2 (Spec.MlDsa.subContract X86_64.abi) := + Verified.of_correct subY_correct subY_ct (by + mldsa_implies [Spec.MlDsa.subContract, Spec.MlDsa.accSig, accK, X86_64.abi, X86_64.argRegs] + [accSat] using accSat) + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YBase.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YBase.lean new file mode 100644 index 000000000..b5f4c278b --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YBase.lean @@ -0,0 +1,130 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay +import VerifiedGarbage.Proof.MlKem.X86_64.YLanes +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Avx2 + +/-! +# ML-DSA on x86-64: coefficients in the lanes of AVX2 registers + +Untrusted: everything here is checked by Lean. The AVX2 code does to each +128-bit lane what the SSE2 code does to a register (`toY`), so the proofs +of the SSE2 code hold of each lane (`ylanes`, ML-KEM's): `YConsts` is +`VConsts` in both lanes (`yconsts_ok`); a 256-bit load of coefficient `j` +puts coefficients `j + 4l` to `j + 4l + 3` in lane `l` (`ylanes_load`), +and a 256-bit store of a register whose lanes hold `a` and `a (· + 4)` +puts `a` at coefficients `j` to `j + 7` (`polyIs_write2Y`, `ylanes_ymm`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (Keep XOnly XKeep YOnly ylanes yld_ok yconst_ok ifp ifn) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt PolyIs) + +/-- `VConsts` in both lanes. -/ +def YConsts (s : State) : Prop := ∀ l < 2, VConsts (s.proj l) + +theorem yonly_yconsts {rs : List XReg} {s s' : State} (h : YOnly rs s s') (hc : YConsts s) + (h14 : XReg.xmm14 ∉ rs) (h15 : XReg.xmm15 ∉ rs) : YConsts s' := fun l hl => + ⟨by rw [State.proj_xmm, h.lane _ h15 l hl]; exact (hc l hl).q, + by rw [State.proj_xmm, h.lane _ h14 l hl]; exact (hc l hl).qinv⟩ + +/-- The constants in both lanes. -/ +theorem yconsts_ok (s : State) : + WP isa (.block yconsts) s fun s' => YConsts s' ∧ Keep [.rax] s s' ∧ s'.mem = s.mem ∧ + s'.mxcsr = s.mxcsr ∧ ∀ r, r ≠ .xmm15 → r ≠ .xmm14 → ∀ l < 2, s'.lane r l = s.lane r l := by + rw [yconsts, WP.block_append_iff] + refine WP.mono (yconst_ok .xmm15 _ s) fun s1 ⟨l1, k1, m1, x1, o1⟩ => + WP.mono (yconst_ok .xmm14 _ s1) fun s2 ⟨l2, k2, m2, x2, o2⟩ => + ⟨fun l hl => ⟨?_, ?_⟩, (k1.trans k2).mono (by simp), m2.trans m1, x2.trans x1, + fun r h15 h14 l hl => by rw [o2 r h14 l hl, o1 r h15 l hl]⟩ + · rw [State.proj_xmm, o2 _ (by decide) l hl, l1 l hl]; decide + · rw [State.proj_xmm, l2 l hl]; decide + +/-- The eight doublewords of a 256-bit value, as those of its lanes. -/ +theorem extract_ymm (hi lo : BitVec 128) {e : Nat} (he : e < 8) : + (hi ++ lo).extractLsb' (32 * e) 32 = if e < 4 then dword lo e else dword hi (e - 4) := by + by_cases h4 : e < 4 + · rw [ifp h4] + apply BitVec.eq_of_getLsbD_eq; intro i hi' + simp only [dword, BitVec.getLsbD_extractLsb', hi', decide_true, Bool.true_and, BitVec.getLsbD_append, + ifp (show 32 * e + i < 128 by omega)] + · rw [ifn h4] + apply BitVec.eq_of_getLsbD_eq; intro i hi' + simp only [dword, BitVec.getLsbD_extractLsb', hi', decide_true, Bool.true_and, BitVec.getLsbD_append, + ifn (show ¬32 * e + i < 128 by omega)] + exact congrArg _ (by omega) + +/-- The doublewords of a 256-bit value are the coefficients `a`. -/ +def YLanes (x : BitVec 256) (a : Nat → Zq) : Prop := ∀ e < 8, (x.extractLsb' (32 * e) 32).toNat = (a e).val + +/-- A register whose lanes hold `a` and `a (· + 4)`. -/ +theorem ylanes_ymm {s : State} {r : XReg} {a : Nat → Zq} (h0 : DLanes (s.lane r 0) a) + (h1 : DLanes (s.lane r 1) (fun e => a (e + 4))) : YLanes (s.ymm r) a := fun e he => by + have h0' : DLanes (s.xmm r) a := h0 + have h1' : DLanes (s.ymmHi r) (fun e => a (e + 4)) := h1 + rw [State.ymm, extract_ymm _ _ he] + split + · exact h0' e (by omega) + · rw [h1' (e - 4) (by omega)]; dsimp only; rw [show e - 4 + 4 = e by omega] + +/-- Coefficient `i` after storing `x` at coefficient `j`. -/ +theorem coeffAt_write256 (m : Mem) (p : Addr) {j : Nat} (hj : j + 8 ≤ 256) (x : BitVec 256) {i : Nat} + (hi : i < 256) : + coeffAt (m.writeW (coeffAddr p j) x) p i = + if j ≤ i ∧ i < j + 8 then x.extractLsb' (32 * (i - j)) 32 else coeffAt m p i := by + split + · rename_i h + rw [coeffAt_eq, show coeffAddr p i = coeffAddr p j + BitVec.ofNat 64 (4 * (i - j)) by + rw [coeffAddr_add, show j + (i - j) = i by omega]] + rw [show 32 * (i - j) = 8 * (4 * (i - j)) by omega] + exact readW_writeW_inside (k := 4 * (i - j)) (n := 4) _ _ _ (by omega) (by decide) + · exact Mem.readW_writeW_sep (Offset.sep p (by omega) (by omega) (by omega)) (by decide) + +/-- Two vectors of eight coefficients stored into a polynomial. -/ +theorem polyIs_write2Y {m : Mem} {p : Addr} {P R : Poly} (hP : PolyIs m p P) {j j' : Nat} + (hj : j + 8 ≤ 256) (hj' : j' + 8 ≤ 256) (hsep : j + 8 ≤ j' ∨ j' + 8 ≤ j) {x y : BitVec 256} + {a b : Nat → Zq} (hx : YLanes x a) (hy : YLanes y b) + (hR : ∀ i < 256, R[i]! = if j ≤ i ∧ i < j + 8 then a (i - j) + else if j' ≤ i ∧ i < j' + 8 then b (i - j') else P[i]!) : + PolyIs ((m.writeW (coeffAddr p j) x).writeW (coeffAddr p j') y) p R := polyIs_of_toNat fun i hi => by + rw [n_eq] at hi + rw [coeffAt_write256 _ _ hj' _ hi, coeffAt_write256 _ _ hj _ hi, hR i hi] + by_cases h1 : j' ≤ i ∧ i < j' + 8 + · rw [ite_eq_left_of_eq_true _ _ (eq_true h1), ite_eq_right_of_eq_false _ _ (eq_false (by omega)), + ite_eq_left_of_eq_true _ _ (eq_true h1)] + exact hy _ (by omega) + · rw [ite_eq_right_of_eq_false _ _ (eq_false h1)] + by_cases h2 : j ≤ i ∧ i < j + 8 + · rw [ite_eq_left_of_eq_true _ _ (eq_true h2), ite_eq_left_of_eq_true _ _ (eq_true h2)] + exact hx _ (by omega) + · rw [ite_eq_right_of_eq_false _ _ (eq_false h2), ite_eq_right_of_eq_false _ _ (eq_false h2), + ite_eq_right_of_eq_false _ _ (eq_false h1)] + exact polyIs_toNat hP (by rw [n_eq]; exact hi) + +theorem pR_contains32 (p : Addr) {j : Nat} (hj : j + 8 ≤ 256) : (pR p).Contains (coeffAddr p j) 32 := + Offset.contains_base p (by omega) (by omega) + +theorem f_in32 {rs : List Region} {fP : Addr} (hw : pR fP ∈ rs) {j : Nat} (hj : j + 8 ≤ 256) : + InRegions rs (coeffAddr fP j) 32 := + ⟨_, hw, pR_contains32 fP hj⟩ + +theorem frame_write2Y {m m' : Mem} {p : Addr} (hf : Frame [pR p] m m') {j j' : Nat} (hj : j + 8 ≤ 256) + (hj' : j' + 8 ≤ 256) (x y : BitVec 256) : + Frame [pR p] m ((m'.writeW (coeffAddr p j) x).writeW (coeffAddr p j') y) := + (hf.writeW (List.mem_singleton_self _) x (pR_contains32 p hj)).writeW (List.mem_singleton_self _) y + (pR_contains32 p hj') + +/-- Lane `l` of a 256-bit load of coefficient `j`. -/ +theorem lane_load {p : Addr} {j l : Nat} : + coeffAddr p j + BitVec.ofNat 64 (16 * l) = coeffAddr p (j + 4 * l) := by + rw [show 16 * l = 4 * (4 * l) by omega, coeffAddr_add] + +/-- The lanes of a 256-bit load of coefficient `j` of `F`. -/ +theorem dlanes_loadY {m : Mem} {p : Addr} {F : Poly} (h : PolyIs m p F) {j : Nat} (hj : j + 8 ≤ 256) + {l : Nat} (hl : l < 2) : + DLanes (m.readW (coeffAddr p j + BitVec.ofNat 64 (16 * l)) 128) (fun e => F[j + 4 * l + e]!) := by + rw [lane_load] + exact dlanes_load h (by omega) + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay.lean new file mode 100644 index 000000000..3468aa875 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay.lean @@ -0,0 +1,632 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YBase +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay21 +import VerifiedGarbage.Proof.MlKem.X86_64.YNtt +import VerifiedGarbage.Proof.MlKem.X86_64.YNttLay + +/-! +# ML-DSA on x86-64: the layers of the NTT and its inverse with `len ≥ 4` on AVX2 registers + +Untrusted: everything here is checked by Lean. For any butterfly code `bf` +that does what `op` does to the doublewords of two SSE registers +(`VBflyOk`) and whose AVX2 form does it in each lane +(`laneSseBlock (toY bf) = some bf`), and any block of the specification +whose butterflies do `op` (`BlkOk`): eight butterflies of a block +(`ystep`), the `len / 8` of them of a block (`yblock_ok`), and the +`128 / len` blocks of a layer with `len ≥ 8` (`ylay_ok`); and the layer +with `len = 4`, two blocks at a time, the lower halves of their +coefficients gathered into `ymm0` and the upper ones into `ymm1` by +`vperm2i128` (`ylay4_ok`). The zetas: `yzeta1_ok` and `yzetaS_ok`, and the +layers' result coefficient by coefficient (`layF_get`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (Keep XOnly XKeep YOnly ylanes yld_ok ifp ifn sel sel_lt sel_zero sel_55 + add_ofNat_zero GOnly addR_ok wp_rcxLoopY wp_countdown ybcast_ok yblend_ok yperm_ok perm20 perm31 + wp_cons_iff lane_setReg lane_setFlags State.setMem_ymm State.setMem_setMem sx32) +open VG.Impl.MlKem.X86_64 (xb xmov toY rcxLoop) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt PolyIs zetas) + +/-! ## A layer, coefficient by coefficient -/ + +section +variable {op : Zq → Zq → Zq → Zq × Zq} {blk : Poly → Nat → Nat → Nat → Nat → Poly} (hblk : BlkOk blk op) +include hblk + +/-- Each coefficient after the first `b` blocks of the layer with `len`. -/ +theorem layF_get (F : Poly) {len : Nat} (hl : 0 < len) (zi : Nat → Nat) {b : Nat} (hb : 2 * len * b ≤ 256) + {j : Nat} (hj : j < 256) : + (layF blk F len zi b)[j]! = if j < 2 * len * b then + (if j % (2 * len) < len then (op F[j]! F[j + len]! (zetas (zi (j / (2 * len))))).1 + else (op F[j - len]! F[j]! (zetas (zi (j / (2 * len))))).2) else F[j]! := by + induction b generalizing j with + | zero => rw [ite_eq_right (by omega)]; rfl + | succ b ih => + have hb' : 2 * len * b + 2 * len ≤ 256 := by rw [Nat.mul_succ] at hb; exact hb + rw [layF, foldl_range_succ, ← layF, + hblk.get _ len _ _ len hl (Nat.le_refl _) (by rw [n_eq]; omega) j (by rw [n_eq]; exact hj)] + have hd : ∀ i, 2 * len * b ≤ i → i < 2 * len * b + 2 * len → i / (2 * len) = b ∧ + i % (2 * len) = i - 2 * len * b := fun i h1 h2 => by + have e1 : i / (2 * len) = b := by + apply Nat.div_eq_of_lt_le + · rw [Nat.mul_comm]; exact h1 + · rw [Nat.succ_mul, Nat.mul_comm b]; exact h2 + refine ⟨e1, ?_⟩ + have := Nat.div_add_mod i (2 * len) + rw [e1] at this; omega + by_cases h1 : 2 * len * b ≤ j ∧ j < 2 * len * b + len + · obtain ⟨d1, d2⟩ := hd j h1.1 (by omega) + rw [ite_eq_left_of_eq_true _ _ (eq_true h1), ih (by omega) hj, ih (by omega) (by omega), d1, + ite_eq_left_of_eq_true _ _ (eq_true (show j < 2 * len * (b + 1) by rw [Nat.mul_succ]; omega)), + ite_eq_left_of_eq_true _ _ (eq_true (show j % (2 * len) < len by omega)), + ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j < 2 * len * b by omega)), + ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j + len < 2 * len * b by omega))] + · rw [ite_eq_right_of_eq_false _ _ (eq_false h1)] + by_cases h2 : 2 * len * b + len ≤ j ∧ j < 2 * len * b + len + len + · obtain ⟨d1, d2⟩ := hd j (by omega) (by omega) + rw [ite_eq_left_of_eq_true _ _ (eq_true h2), ih (by omega) (by omega), ih (by omega) hj, d1, + ite_eq_left_of_eq_true _ _ (eq_true (show j < 2 * len * (b + 1) by rw [Nat.mul_succ]; omega)), + ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j % (2 * len) < len by omega)), + ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j - len < 2 * len * b by omega)), + ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j < 2 * len * b by omega))] + · rw [ite_eq_right_of_eq_false _ _ (eq_false h2), ih (by omega) hj] + by_cases h3 : j < 2 * len * b + · rw [ite_eq_left_of_eq_true _ _ (eq_true h3), + ite_eq_left_of_eq_true _ _ (eq_true (show j < 2 * len * (b + 1) by rw [Nat.mul_succ]; omega))] + · rw [ite_eq_right_of_eq_false _ _ (eq_false h3), + ite_eq_right_of_eq_false _ _ (eq_false (show ¬ j < 2 * len * (b + 1) by rw [Nat.mul_succ]; omega))] + +end + +/-! ## Coefficients in both lanes -/ + +/-- The facts a piece of a layer keeps. -/ +structure BInvY (fP : Addr) (s₀ s : State) : Prop where + keep : Keep [.r8, .rcx, .rdx, .rax] s₀ s + frame : Frame [pR fP] s₀.mem s.mem + consts : YConsts s + mxcsr : s.mxcsr = s₀.mxcsr + +theorem BInvY.trans {fP : Addr} {s₁ s₂ s₃ : State} (h₁ : BInvY fP s₁ s₂) (h₂ : BInvY fP s₂ s₃) : + BInvY fP s₁ s₃ := + ⟨(h₁.keep.trans h₂.keep).mono (by simp), h₁.frame.trans h₂.frame, h₂.consts, h₂.mxcsr.trans h₁.mxcsr⟩ + +/-- `YConsts` after code that changed the vector registers `rs` only, then +nothing in them. -/ +theorem ylanes_gpr {s s' : State} (h : ∀ r l, s'.lane r l = s.lane r l) {rs : List XReg} {s₀ : State} + (o : YOnly rs s₀ s) (hc : YConsts s₀) (h14 : XReg.xmm14 ∉ rs) (h15 : XReg.xmm15 ∉ rs) : YConsts s' := + fun l hl => ⟨by rw [State.proj_xmm, h]; exact (yonly_yconsts o hc h14 h15 l hl).q, + by rw [State.proj_xmm, h]; exact (yonly_yconsts o hc h14 h15 l hl).qinv⟩ + +/-- Two registers stored into a polynomial at coefficients `j` and `j'`, +lane `l` of each holding the four coefficients of `R` from `j + 4l` and +`j' + 4l`. -/ +theorem polyIs_write2L {m : Mem} {p : Addr} {P R : Poly} (hP : PolyIs m p P) {j j' : Nat} + (hj : j + 8 ≤ 256) (hj' : j' + 8 ≤ 256) (hsep : j + 8 ≤ j' ∨ j' + 8 ≤ j) {s : State} {x y : XReg} + (hx : ∀ l < 2, DLanes (s.lane x l) (fun e => R[j + 4 * l + e]!)) + (hy : ∀ l < 2, DLanes (s.lane y l) (fun e => R[j' + 4 * l + e]!)) + (hR : ∀ i < 256, (i < j ∨ j + 8 ≤ i) → (i < j' ∨ j' + 8 ≤ i) → R[i]! = P[i]!) : + PolyIs ((m.writeW (coeffAddr p j) (s.ymm x)).writeW (coeffAddr p j') (s.ymm y)) p R := + polyIs_write2Y hP hj hj' hsep (a := fun e => R[j + e]!) (b := fun e => R[j' + e]!) + (ylanes_ymm (fun e he => (hx 0 (by decide) e he).trans (by simp only [Nat.mul_zero, Nat.add_zero])) + (fun e he => (hx 1 (by decide) e he).trans (by dsimp only; rw [show j + 4 * 1 + e = j + (e + 4) by omega]))) + (ylanes_ymm (fun e he => (hy 0 (by decide) e he).trans (by simp only [Nat.mul_zero, Nat.add_zero])) + (fun e he => (hy 1 (by decide) e he).trans (by dsimp only; rw [show j' + 4 * 1 + e = j' + (e + 4) by omega]))) + fun i hi => by + by_cases h1 : j ≤ i ∧ i < j + 8 + · rw [ite_eq_left_of_eq_true _ _ (eq_true h1), show j + (i - j) = i by omega] + · rw [ite_eq_right_of_eq_false _ _ (eq_false h1)] + by_cases h2 : j' ≤ i ∧ i < j' + 8 + · rw [ite_eq_left_of_eq_true _ _ (eq_true h2), show j' + (i - j') = i by omega] + · rw [ite_eq_right_of_eq_false _ _ (eq_false h2)]; exact hR i hi (by omega) (by omega) + +theorem DLanes.congr {x : BitVec 128} {f g : Nat → Zq} (h : DLanes x f) (e : ∀ i < 4, f i = g i) : DLanes x g := + fun i hi => by rw [h i hi, e i hi] + +theorem ZLanes.congr {x : BitVec 128} {f g : Nat → Zq} (h : ZLanes x f) (e : ∀ i < 4, f i = g i) : ZLanes x g := + fun i hi => by rw [h i hi, e i hi] + +theorem getP_congr (P : Poly) {a b : Nat} (h : a = b) : P[a]! = P[b]! := h ▸ rfl + +/-- A butterfly of the coefficients at `a` and `b` of `P` with the zeta of +index `zi c`, at other indices that are the same. -/ +theorem op_idx (op : Zq → Zq → Zq → Zq × Zq) (P : Poly) (zi : Nat → Nat) {a b c a' b' c' : Nat} (ha : a = a') + (hb : b = b') (hc : c = c') : op P[a]! P[b]! (zetas (zi c)) = op P[a']! P[b']! (zetas (zi c')) := by + subst ha hb hc; rfl + +/-- The prologue of the layers with `len` = 4, 2 and 1. -/ +theorem ypre21 {fP sP : Addr} (k : Nat) (hk : k + 4 ≤ 256) {s : State} (hdi : s.gpr .rdi = fP) + (hsi : s.gpr .rsi = sP) : + WP isa (.block (([.mov .rdx (.reg .rdi)] : List Instr) ++ leaR .r8 .rsi (4 * k))) s fun w => + w.gpr .rdx = fP ∧ w.gpr .r8 = coeffAddr sP k ∧ GOnly [.rdx, .r8] s w ∧ w.ymmHi = s.ymmHi := by + simp only [leaR] + vrund [sx_ofNat (show 4 * k < 2 ^ 31 by omega), hsi, hdi] + exact ⟨by gonlyd, rfl⟩ + +/-! ## The zetas -/ + +theorem zodd_F5 (x : BitVec 128) : ZOdd x (shufDwords x 0xF5) := fun j hj => by + rw [dword_shufDwords _ _ (by omega)] + rcases (by omega : j = 0 ∨ j = 1) with rfl | rfl <;> rfl + +/-- The zetas `pshufd` with `o` leaves of the four at index `k` of the table. -/ +theorem zlanes_tab (o : BitVec 8) {zP : Addr} {k : Nat} (hk : ∀ j < 4, k + sel o j < 256) {m : Mem} + (ht : Tab zmTab m zP 256) : + ZLanes (shufDwords (m.readW (coeffAddr zP k) 128) o) (fun i => zetas (k + sel o i)) := fun i hi => by + have hs := sel_lt o i + rw [dword_shufDwords_sel _ _ hi, dword_readW _ _ hs, coeffAddr_add] + exact tab_zeta ht (hk i hi) + +theorem zsse1_ok (t : State) : + WP isa (.block [.xop (.pshufd .xmm13 .xmm13 0), .xop (.pshufd .xmm12 .xmm13 0xF5)]) t fun t' => + (t'.xmm .xmm13 = shufDwords (t.xmm .xmm13) 0 ∧ + t'.xmm .xmm12 = shufDwords (shufDwords (t.xmm .xmm13) 0) 0xF5) ∧ XOnly [.xmm13, .xmm12] t t' := by + vrun + exact ⟨by first | trivial | simp, by xonly⟩ + +theorem yzeta1_ok {zP : Addr} {k : Nat} (hk : k + 4 ≤ 256) {s : State} (h8 : s.gpr .r8 = coeffAddr zP k) + (hin : InRegions (s.rd ++ s.wr) (coeffAddr zP k) 16) (ht : Tab zmTab s.mem zP 256) : + WP isa (.block yzeta1) s fun s' => (∀ l < 2, ZLanes (s'.lane .xmm13 l) (fun _ => zetas k) ∧ + ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧ YOnly [.xmm13, .xmm12] s s' := by + rw [yzeta1, wp_cons_iff] + refine WP.mono (ybcast_ok (by rw [h8, add_ofNat_zero]; exact hin)) fun s1 ⟨b1, o1⟩ => ?_ + refine WP.mono (ylanes (by decide) (P := fun l t => t.xmm .xmm13 = shufDwords (s1.lane .xmm13 l) 0 ∧ + t.xmm .xmm12 = shufDwords (shufDwords (s1.lane .xmm13 l) 0) 0xF5) + fun l _ => zsse1_ok (s1.proj l)) fun s2 ⟨l2, o2⟩ => ⟨fun l hl => ?_, (o1.trans o2).mono (by simp)⟩ + have e13 : s2.lane .xmm13 l = _ := (l2 l hl).1 + have e12 : s2.lane .xmm12 l = _ := (l2 l hl).2 + rw [e12, e13, b1 l hl, h8, add_ofNat_zero] + refine ⟨fun i hi => ?_, zodd_F5 _⟩ + rw [zlanes_tab 0 (fun j _ => by rw [sel_zero]; omega) ht i hi] + dsimp only; rw [sel_zero, Nat.add_zero] + +theorem zsseS_ok (o₀ o₁ : BitVec 8) (t : State) : + WP isa (.block [.xop (.pshufd .xmm2 .xmm13 o₁), .xop (.pshufd .xmm13 .xmm13 o₀)]) t fun t' => + (t'.xmm .xmm13 = shufDwords (t.xmm .xmm13) o₀ ∧ t'.xmm .xmm2 = shufDwords (t.xmm .xmm13) o₁) ∧ + XOnly [.xmm2, .xmm13] t t' := by + vrun + exact ⟨by first | trivial | simp, by xonly⟩ + +theorem zsseF5_ok (t : State) : + WP isa (.block [.xop (.pshufd .xmm12 .xmm13 0xF5)]) t fun t' => + t'.xmm .xmm12 = shufDwords (t.xmm .xmm13) 0xF5 ∧ XOnly [.xmm12] t t' := by + vrun + exact ⟨by first | trivial | simp, by xonly⟩ + +/-- The odd zetas of each lane of `ymm13` in the even doublewords of `ymm12`. -/ +theorem yF5_ok (s : State) : + WP isa (.block (toY [.xop (.pshufd .xmm12 .xmm13 0xF5)])) s fun s' => + (∀ l < 2, s'.lane .xmm12 l = shufDwords (s.lane .xmm13 l) 0xF5) ∧ YOnly [.xmm12] s s' := + ylanes (by decide) (P := fun l t => t.xmm .xmm12 = shufDwords (s.lane .xmm13 l) 0xF5) + fun l _ => zsseF5_ok (s.proj l) + +theorem yzetaS_ok (o₀ o₁ : BitVec 8) {zP : Addr} {k : Nat} (hk0 : ∀ j < 4, k + sel o₀ j < 256) + (hk1 : ∀ j < 4, k + sel o₁ j < 256) {s : State} (h8 : s.gpr .r8 = coeffAddr zP k) + (hin : InRegions (s.rd ++ s.wr) (coeffAddr zP k) 16) (ht : Tab zmTab s.mem zP 256) : + WP isa (.block (yzetaS o₀ o₁)) s fun s' => ZLanes (s'.lane .xmm13 0) (fun i => zetas (k + sel o₀ i)) ∧ + ZLanes (s'.lane .xmm13 1) (fun i => zetas (k + sel o₁ i)) ∧ + (∀ l < 2, ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧ YOnly [.xmm13, .xmm2, .xmm12] s s' := by + rw [yzetaS, WP.block_append_iff, WP.block_append_iff, wp_cons_iff] + refine WP.mono (ybcast_ok (by rw [h8, add_ofNat_zero]; exact hin)) fun s1 ⟨b1, o1⟩ => ?_ + refine WP.mono (ylanes (by rfl) (P := fun l t => + t.xmm .xmm13 = shufDwords (s1.lane .xmm13 l) o₀ ∧ t.xmm .xmm2 = shufDwords (s1.lane .xmm13 l) o₁) + fun l _ => zsseS_ok o₀ o₁ (s1.proj l)) fun s2 ⟨l2, o2⟩ => ?_ + refine WP.mono (yblend_ok s2) fun s3 ⟨e0, e1, o3⟩ => ?_ + refine WP.mono (yF5_ok s3) fun s4 ⟨f4, o4⟩ => ?_ + have x0 : s4.lane .xmm13 0 = shufDwords (s.mem.readW (coeffAddr zP k) 128) o₀ := by + have e : s2.lane .xmm13 0 = _ := (l2 0 (by decide)).1 + rw [o4.lane _ (by decide) 0 (by decide), e0, e, b1 0 (by decide), h8, add_ofNat_zero] + have x1 : s4.lane .xmm13 1 = shufDwords (s.mem.readW (coeffAddr zP k) 128) o₁ := by + have e : s2.lane .xmm2 1 = _ := (l2 1 (by decide)).2 + rw [o4.lane _ (by decide) 1 (by decide), e1, e, b1 1 (by decide), h8, add_ofNat_zero] + refine ⟨by rw [x0]; exact zlanes_tab o₀ hk0 ht, by rw [x1]; exact zlanes_tab o₁ hk1 ht, + fun l hl => ?_, (((o1.trans o2).trans o3).trans o4).mono (by simp)⟩ + rw [f4 l hl, o4.lane _ (by decide) l hl] + exact zodd_F5 _ + +/-! ## A layer with `len ≥ 8` -/ + +section +variable {bf : List Instr} {op : Zq → Zq → Zq → Zq × Zq} (hbf : VBflyOk bf op) + (hY : laneSseBlock (toY bf) = some bf) + {blk : Poly → Nat → Nat → Nat → Nat → Poly} (hblk : BlkOk blk op) +include hbf hY + +/-- The butterflies of `bf` in each lane, from the coefficients `x` and `y` +of the lanes of `ymm0` and `ymm1` and the zetas `ζ` of those of `ymm13`. -/ +theorem ybf_ok {s : State} (hc : YConsts s) {x y ζ : Nat → Nat → Zq} + (hx : ∀ l < 2, DLanes (s.lane .xmm0 l) (x l)) (hy : ∀ l < 2, DLanes (s.lane .xmm1 l) (y l)) + (hz : ∀ l < 2, ZLanes (s.lane .xmm13 l) (ζ l)) (ho : ∀ l < 2, ZOdd (s.lane .xmm13 l) (s.lane .xmm12 l)) : + WP isa (.block (toY bf)) s fun s' => + (∀ l < 2, DLanes (s'.lane .xmm0 l) (fun i => (op (x l i) (y l i) (ζ l i)).1) ∧ + DLanes (s'.lane .xmm3 l) (fun i => (op (x l i) (y l i) (ζ l i)).2)) ∧ + YOnly [.xmm1, .xmm2, .xmm4, .xmm0, .xmm3] s s' := + ylanes hY (P := fun l t => DLanes (t.xmm .xmm0) (fun i => (op (x l i) (y l i) (ζ l i)).1) ∧ + DLanes (t.xmm .xmm3) (fun i => (op (x l i) (y l i) (ζ l i)).2)) + fun l hl => WP.mono (hbf _ (hc l hl) _ _ _ (hx l hl) (hy l hl) (hz l hl) (ho l hl)) + fun _ ⟨a, b, c⟩ => ⟨⟨a, b⟩, c⟩ + +include hblk + +/-- The body of the loop over the vectors of a block. -/ +abbrev ybody (bf : List Instr) (len : Nat) : List Instr := + [.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm1 (at_ .rdx (4 * len))] ++ toY bf ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx (4 * len)) .xmm3, + .alu .add .rdx (.imm 32)] ++ [.alu .sub .rcx (.imm 1)] + +theorem ystep {fP : Addr} {len st u k : Nat} (hl : 8 ≤ len) (hs : st + 2 * len ≤ 256) (hu : 8 * u + 8 ≤ len) + {G : Poly} {s : State} (hc : YConsts s) (hz : ∀ l < 2, ZLanes (s.lane .xmm13 l) (fun _ => zetas k)) + (ho : ∀ l < 2, ZOdd (s.lane .xmm13 l) (s.lane .xmm12 l)) + (hdx : s.gpr .rdx = coeffAddr fP (st + 8 * u)) (hS : PolyIs s.mem fP (blk G len k st (8 * u))) + (hw : pR fP ∈ s.wr) : + WP isa (.block (ybody bf len)) s fun s' => + PolyIs s'.mem fP (blk G len k st (8 * (u + 1))) ∧ s'.gpr .rdx = coeffAddr fP (st + 8 * (u + 1)) ∧ + Frame [pR fP] s.mem s'.mem ∧ YConsts s' ∧ (∀ l < 2, s'.lane .xmm13 l = s.lane .xmm13 l) ∧ + (∀ l < 2, s'.lane .xmm12 l = s.lane .xmm12 l) ∧ Keep [.rdx, .rcx] s s' ∧ + s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) ∧ s'.mxcsr = s.mxcsr := by + have j0 : st + 8 * u + 8 ≤ 256 := by omega + have j1 : st + 8 * u + len + 8 ≤ 256 := by omega + have a1 : coeffAddr fP (st + 8 * u) + BitVec.ofNat 64 (4 * len) = coeffAddr fP (st + 8 * u + len) := + coeffAddr_add _ _ _ + have r0 : InRegions (s.rd ++ s.wr) (coeffAddr fP (st + 8 * u) + BitVec.ofNat 64 0) 32 := by + rw [add_ofNat_zero]; exact f_in32 (List.mem_append_right _ hw) j0 + have r1 : InRegions (s.rd ++ s.wr) (coeffAddr fP (st + 8 * u) + BitVec.ofNat 64 (4 * len)) 32 := by + rw [a1]; exact f_in32 (List.mem_append_right _ hw) j1 + rw [ybody, List.append_assoc, List.append_assoc, WP.block_append_iff, + show ∀ a b : Instr, [a, b] = [a] ++ [b] from fun _ _ => rfl, WP.block_append_iff] + refine WP.mono (yld_ok (by rw [hdx]; exact r0)) fun s1 ⟨L1, o1⟩ => ?_ + refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr, hdx]; exact r1)) fun s2 ⟨L2, o2⟩ => ?_ + rw [WP.block_append_iff] + have o12 := o1.trans o2 + generalize hP : blk G len k st (8 * u) = P at hS + refine WP.mono (ybf_ok hbf hY (yonly_yconsts o12 hc (by decide) (by decide)) + (x := fun l e => P[st + 8 * u + 4 * l + e]!) (y := fun l e => P[st + 8 * u + len + 4 * l + e]!) + (ζ := fun _ _ => zetas k) + (fun l hl => by + rw [o2.lane _ (by decide) l hl, L1 l hl, hdx, add_ofNat_zero]; exact dlanes_loadY hS j0 hl) + (fun l hl => by + rw [L2 l hl, o1.gpr, o1.mem, hdx, a1]; exact dlanes_loadY hS j1 hl) + (fun l hl => by rw [o12.lane _ (by decide) l hl]; exact hz l hl) + (fun l hl => by rw [o12.lane _ (by decide) l hl, o12.lane _ (by decide) l hl]; exact ho l hl)) + fun s3 ⟨B3, o3⟩ => ?_ + have o13 := o12.trans o3 + have w0 : InRegions s3.wr (s3.gpr .rdx) 32 := by + rw [o13.wr, o13.gpr, hdx]; exact f_in32 hw j0 + have w1 : InRegions s3.wr (s3.gpr .rdx + BitVec.ofNat 64 (4 * len)) 32 := by + rw [o13.wr, o13.gpr, hdx, a1]; exact f_in32 hw j1 + vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, State.setMem_ymm, + State.setMem_setMem, w0, w1, sx32] + have g3 : s3.gpr .rdx = coeffAddr fP (st + 8 * u) := by rw [o13.gpr, hdx] + rw [g3, a1, o13.mem] + refine ⟨?_, ?_, ?_, ?_, ?_, ?_, ⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, ?_⟩ + · have hR : ∀ i < 256, (blk G len k st (8 * (u + 1)))[i]! = if st + 8 * u ≤ i ∧ i < st + 8 * u + 8 then + (op P[i]! P[i + len]! (zetas k)).1 else if st + 8 * u + len ≤ i ∧ i < st + 8 * u + len + 8 then + (op P[i - len]! P[i]! (zetas k)).2 else P[i]! := fun i hi => by + rw [← hP, show 8 * (u + 1) = 8 * u + 8 by omega, hblk.add, hblk.get _ _ _ _ _ (by omega) (by omega) + (by rw [n_eq]; omega) _ (by rw [n_eq]; exact hi)] + refine polyIs_write2L (s := s3) hS j0 j1 (by omega) (fun l hl e he => ?_) (fun l hl e he => ?_) + (fun i hi h1 h2 => by rw [hR i hi, ite_eq_right (by omega), ite_eq_right (by omega)]) + · rw [(B3 l hl).1 e he]; dsimp only; rw [hR _ (by omega), ite_eq_left (by omega), + show st + 8 * u + len + 4 * l + e = st + 8 * u + 4 * l + e + len by omega] + · rw [(B3 l hl).2 e he]; dsimp only; rw [hR _ (by omega), ite_eq_right (by omega), ite_eq_left (by omega), + show st + 8 * u + len + 4 * l + e - len = st + 8 * u + 4 * l + e by omega] + · rw [show (32 : BitVec 64) = BitVec.ofNat 64 (4 * 8) from rfl, coeffAddr_add, + show st + 8 * u + 8 = st + 8 * (u + 1) by omega] + · exact frame_write2Y (Frame.refl _ _) j0 j1 _ _ + · exact ylanes_gpr (s := s3) (fun r l => by simp only [lane_setReg, lane_setFlags, State.setMem_lane]) o13 hc + (by decide) (by decide) + · intro l hl; simp only [lane_setReg, lane_setFlags, State.setMem_lane]; exact o13.lane _ (by decide) l hl + · intro l hl; simp only [lane_setReg, lane_setFlags, State.setMem_lane]; exact o13.lane _ (by decide) l hl + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr.1, hr.2, ite_false, State.setMem_gpr] + rw [o13.gpr] + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; rw [o13.rd] + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; rw [o13.wr] + · rw [o13.gpr] + · rw [o13.gpr] + · exact o13.mxcsr + +/-- The code of a block of a layer with `len ≥ 8`. -/ +abbrev yblk (bf : List Instr) (len : Nat) (dz : BitVec 32) : Prog isa := + .seq (.block (yzeta1 ++ [.alu .add .r8 (.imm dz)])) + (.seq (rcxLoop (len / 8) ([.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), + .vmovdquLoad .l256 .xmm1 (at_ .rdx (4 * len))] ++ toY bf ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx (4 * len)) .xmm3, + .alu .add .rdx (.imm 32)])) + (.block [.alu .add .rdx (.imm (BitVec.ofNat 32 (4 * len))), .alu .sub .rax (.imm 1)])) + +theorem yblock_ok {fP sP : Addr} {len st kz : Nat} (h8 : 8 ≤ len) (hl8 : len % 8 = 0) (hl : len ≤ 128) + (hs : st + 2 * len ≤ 256) (hkz : kz + 4 ≤ 256) (dz : BitVec 32) {G : Poly} {s : State} (hc : YConsts s) + (hdx : s.gpr .rdx = coeffAddr fP st) (h8r : s.gpr .r8 = coeffAddr sP kz) (hS : PolyIs s.mem fP G) + (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (yblk bf len dz) s fun s' => PolyIs s'.mem fP (blk G len kz st len) ∧ + s'.gpr .rdx = coeffAddr fP (st + 2 * len) ∧ s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ + s'.gpr .rax = s.gpr .rax - 1 ∧ s'.zf = some (s.gpr .rax - 1 == 0) ∧ BInvY fP s s' := by + -- the zeta + refine WP.seq ?_ + rw [WP.block_append_iff] + refine WP.mono (yzeta1_ok hkz h8r (tab_in (List.mem_append_right _ hw) hkz) hT) fun s1 ⟨z1, o1⟩ => ?_ + have g1 : s1.gpr = s.gpr := o1.gpr + refine WP.mono (addR_ok .r8 dz s1) fun s2 ⟨h82, o2, y2⟩ => ?_ + have l2 := o2.lane y2 + have c2 : YConsts s2 := ylanes_gpr (s := s1) l2 o1 hc (by decide) (by decide) + have dx2 : s2.gpr .rdx = coeffAddr fP st := by rw [o2.keep.gpr (by decide), g1, hdx] + have hw2 : pR fP ∈ s2.wr := by rw [o2.keep.2.2, o1.wr]; exact hwf + have m2 : s2.mem = s.mem := by rw [o2.mem, o1.mem] + refine WP.seq (WP.mono (wp_rcxLoopY (N := len / 8) (by omega) (by omega) + (fun u w => PolyIs w.mem fP (blk G len kz st (8 * u)) ∧ w.gpr .rdx = coeffAddr fP (st + 8 * u) ∧ + YConsts w ∧ (∀ l < 2, w.lane .xmm13 l = s2.lane .xmm13 l) ∧ (∀ l < 2, w.lane .xmm12 l = s2.lane .xmm12 l) ∧ + Keep [.rcx, .rdx] s2 w ∧ Frame [pR fP] s2.mem w.mem ∧ w.mxcsr = s2.mxcsr) + (fun w o hy _ => ⟨by rw [hblk.zero, o.mem, m2]; exact hS, by rw [o.keep.gpr (by decide), dx2]; rfl, + ylanes_gpr (s := s2) (o.lane hy) (YOnly.refl [] s2) c2 (by decide) (by decide), + fun l _ => o.lane hy _ l, fun l _ => o.lane hy _ l, o.keep.mono (by simp), + by rw [o.mem]; exact Frame.refl _ _, o.mxcsr⟩) + (fun u hu w ⟨hS', hdx', hc', hz', hzo', hk', hf', hx'⟩ => WP.mono (ystep hbf hY hblk h8 hs (by + have := Nat.div_mul_cancel (Nat.dvd_of_mod_eq_zero hl8); omega) hc' + (fun l hl => by rw [hz' l hl, l2]; exact (z1 l hl).1) + (fun l hl => by rw [hz' l hl, hzo' l hl, l2, l2]; exact (z1 l hl).2) hdx' hS' + (by rw [hk'.2.2]; exact hw2)) + fun w' ⟨hS'', hdx'', hf'', hc'', hz'', hzo'', hk'', hcx, hzf, hx''⟩ => + ⟨⟨hS'', hdx'', hc'', fun l hl => by rw [hz'' l hl, hz' l hl], fun l hl => by rw [hzo'' l hl, hzo' l hl], + (hk'.trans hk'').mono (by simp), hf'.trans hf'', by rw [hx'', hx']⟩, hcx, hzf⟩)) + fun w ⟨hS3, hdx3, hc3, _, _, hk3, hf3, hx3⟩ => ?_) + rw [show 8 * (len / 8) = len from Nat.mul_div_cancel' (Nat.dvd_of_mod_eq_zero hl8)] at hS3 hdx3 + have hax : w.gpr .rax = s.gpr .rax := by rw [hk3.gpr (by decide), o2.keep.gpr (by decide), g1] + have h8w : w.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz := by rw [hk3.gpr (by decide), h82, g1] + vrund [hdx3, sx_ofNat (show 4 * len < 2 ^ 31 by omega), hax, h8w] + refine ⟨hS3, by rw [coeffAddr_add, show st + len + len = st + 2 * len by omega], ?_⟩ + have k1 : Keep [.r8, .rcx, .rdx, .rax] s w := + (Keep.trans (⟨fun r _ => by rw [g1], o1.rd, o1.wr⟩ : Keep [] s s1) (o2.keep.trans hk3)).mono (by simp) + refine ⟨⟨fun r hr => ?_, k1.2.1, k1.2.2⟩, by rw [← m2]; exact hf3, + ylanes_gpr (s := w) (fun r l => by simp only [lane_setReg, lane_setFlags]) (YOnly.refl [] w) hc3 + (by decide) (by decide), + by simp only [RegUpd.mxcsr_setReg, RegUpd.mxcsr_setFlags]; rw [hx3, o2.mxcsr, o1.mxcsr]⟩ + simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false] + exact k1.gpr (by simp [hr]) + +theorem ylay_ok {fP sP : Addr} {len k : Nat} (hlen : len ∈ [8, 16, 32, 64, 128]) (dz : BitVec 32) + (zi : Nat → Nat) (hz0 : zi 0 = k) (hzi : ∀ c < 128 / len, zi c + 4 ≤ 256) + (hstep : ∀ c < 128 / len, coeffAddr sP (zi c) + BitVec.signExtend 64 dz = coeffAddr sP (zi (c + 1))) + {F : Poly} {s : State} (hc : YConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) + (hd : (pR sP).Disjoint (pR fP)) : + WP isa (ylay bf len k dz) s fun s' => PolyIs s'.mem fP (layF blk F len zi (128 / len)) ∧ + BInvY fP s s' := by + have hl : 8 ≤ len ∧ len % 8 = 0 ∧ len ≤ 128 ∧ 2 * len * (128 / len) = 256 ∧ 0 < 128 / len ∧ + 128 / len ≤ 16 := by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hlen + rcases hlen with rfl | rfl | rfl | rfl | rfl <;> decide + obtain ⟨h8, hl8, hl128, hcov, hpos, h16⟩ := hl + have hk : k + 4 ≤ 256 := hz0 ▸ hzi 0 hpos + refine WP.seq (WP.mono (Q := fun (w : State) => w.gpr .rdx = fP ∧ w.gpr .r8 = coeffAddr sP k ∧ + w.gpr .rax = BitVec.ofNat 64 (128 / len) ∧ GOnly [.rdx, .r8, .rax] s w ∧ w.ymmHi = s.ymmHi) + (by + simp only [leaR] + vrund [sx_ofNat (show 4 * k < 2 ^ 31 by omega), hsi, hdi, RegUpd.ymmHi_setReg] + refine ⟨?_, by gonlyd, rfl⟩ + apply BitVec.eq_of_toNat_eq + rw [BitVec.toNat_setWidth, BitVec.toNat_ofNat, BitVec.toNat_ofNat] + omega) fun w ⟨hdx, h8r, hax, o, hy⟩ => ?_) + have hwf' : pR fP ∈ w.wr := by rw [o.keep.2.2]; exact hwf + have hw' : pR sP ∈ w.wr := by rw [o.keep.2.2]; exact hw + have cw : YConsts w := ylanes_gpr (s := s) (o.lane hy) (YOnly.refl [] s) hc (by decide) (by decide) + refine WP.mono (wp_countdown (cnt := .rax) (N := 128 / len) (by omega) hpos + (fun c u => PolyIs u.mem fP (layF blk F len zi c) ∧ u.gpr .rdx = coeffAddr fP (2 * len * c) ∧ + u.gpr .r8 = coeffAddr sP (zi c) ∧ BInvY fP w u ∧ Tab zmTab u.mem sP 256) + (fun c hc u ⟨hS', hdx', h8', hb', hT'⟩ _ => ?_) (fun u h => h) + ⟨by rw [o.mem]; exact hS, by rw [hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [h8r, hz0], ⟨Keep.refl _ _, Frame.refl _ _, cw, rfl⟩, by rw [o.mem]; exact hT⟩ hax) + fun u ⟨hS', _, _, hb', _⟩ => ⟨hS', ⟨(o.keep.trans hb'.keep).mono (by simp), + by rw [← o.mem]; exact hb'.frame, hb'.consts, by rw [hb'.mxcsr, o.mxcsr]⟩⟩ + have hs : 2 * len * c + 2 * len ≤ 256 := by + have : 2 * len * (c + 1) ≤ 2 * len * (128 / len) := Nat.mul_le_mul_left _ (by omega) + rw [Nat.mul_succ] at this; omega + refine WP.mono (yblock_ok hbf hY hblk h8 hl8 hl128 hs (hzi c hc) dz hb'.consts hdx' h8' hS' hT' + (by rw [hb'.keep.2.2]; exact hwf') (by rw [hb'.keep.2.2]; exact hw')) + fun u' ⟨hS'', hdx'', h8'', hax'', hzf'', hb''⟩ => + ⟨⟨by rw [layF, foldl_range_succ]; exact hS'', by rw [hdx'', Nat.mul_succ], + by rw [h8'', h8', hstep c hc], hb'.trans hb'', + hT'.frame hb''.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide)⟩, hax'', hzf''⟩ + +/-! ## The layer with `len = 4` -/ + +/-- The body of the layer with `len = 4`. -/ +abbrev ybody4 (bf : List Instr) (o₀ o₁ : BitVec 8) (dz : BitVec 32) : List Instr := + [.vmovdquLoad .l256 .xmm4 (at_ .rdx 0)] ++ ([.vmovdquLoad .l256 .xmm5 (at_ .rdx 32)] ++ (yzetaS o₀ o₁ ++ + ([.alu .add .r8 (.imm dz)] ++ ([.vop (.vperm2i128 .xmm0 .xmm4 .xmm5 0x20)] ++ + ([.vop (.vperm2i128 .xmm1 .xmm4 .xmm5 0x31)] ++ (toY bf ++ ([.vop (.vperm2i128 .xmm4 .xmm0 .xmm3 0x20)] ++ + ([.vop (.vperm2i128 .xmm5 .xmm0 .xmm3 0x31)] ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm4, .vmovdquStore .l256 (at_ .rdx 32) .xmm5, .alu .add .rdx (.imm 64), + .alu .sub .rcx (.imm 1)])))))))) + +theorem ystep4 {fP sP : Addr} {m kb : Nat} (hm : m < 16) (o₀ o₁ : BitVec 8) (dz : BitVec 32) (zi : Nat → Nat) + (hk : kb + 4 ≤ 256) (hsel : ∀ e < 4, kb + sel o₀ e = zi (2 * m) ∧ kb + sel o₁ e = zi (2 * m + 1)) + {F : Poly} {s : State} (hc : YConsts s) (hdx : s.gpr .rdx = coeffAddr fP (16 * m)) + (h8 : s.gpr .r8 = coeffAddr sP kb) (hS : PolyIs s.mem fP (layF blk F 4 zi (2 * m))) + (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (.block (ybody4 bf o₀ o₁ dz)) s fun s' => + PolyIs s'.mem fP (layF blk F 4 zi (2 * (m + 1))) ∧ s'.gpr .rdx = coeffAddr fP (16 * (m + 1)) ∧ + s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ + s'.zf = some (s.gpr .rcx - 1 == 0) ∧ BInvY fP s s' := by + have j0 : 16 * m + 8 ≤ 256 := by omega + have j1 : 16 * m + 8 + 8 ≤ 256 := by omega + have a1 : coeffAddr fP (16 * m) + BitVec.ofNat 64 32 = coeffAddr fP (16 * m + 8) := coeffAddr_add _ _ 8 + have r0 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 0) 32 := by + rw [hdx, add_ofNat_zero]; exact f_in32 (List.mem_append_right s.rd hwf) j0 + have r1 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 32) 32 := by + rw [hdx, a1]; exact f_in32 (List.mem_append_right s.rd hwf) j1 + generalize hP : layF blk F 4 zi (2 * m) = P at hS + -- the loads + rw [ybody4, WP.block_append_iff] + refine WP.mono (yld_ok r0) fun s1 ⟨L4, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr]; exact r1)) fun s2 ⟨L5, o2⟩ => ?_ + have o12 := o1.trans o2 + -- the zetas + rw [WP.block_append_iff] + have hk0 : ∀ j < 4, kb + sel o₀ j < 256 := fun j _ => by have := sel_lt o₀ j; omega + have hk1 : ∀ j < 4, kb + sel o₁ j < 256 := fun j _ => by have := sel_lt o₁ j; omega + refine WP.mono (yzetaS_ok o₀ o₁ (zP := sP) (k := kb) hk0 hk1 (by rw [o12.gpr, h8]) + (by rw [o12.rd, o12.wr]; exact tab_in (List.mem_append_right _ hw) hk) (by rw [o12.mem]; exact hT)) + fun s3 ⟨Z0, Z1, ZO, o3⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (addR_ok .r8 dz s3) fun s4 ⟨h84, g4, y4⟩ => ?_ + have l4 := g4.lane y4 + -- the lower and upper halves + rw [WP.block_append_iff] + refine WP.mono (yperm_ok s4) fun s5 ⟨P5, o5⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (yperm_ok s5) fun s6 ⟨P6, o6⟩ => ?_ + have c6 : YConsts s6 := yonly_yconsts (o5.trans o6) + (ylanes_gpr (s := s3) l4 (o12.trans o3) hc (by decide) (by decide)) (by decide) (by decide) + have m3 : s3.mem = s.mem := (o12.trans o3).mem + have q4 : ∀ l < 2, s4.lane .xmm4 l = s.mem.readW (coeffAddr fP (16 * m) + BitVec.ofNat 64 (16 * l)) 128 := + fun l hl => by rw [l4, o3.lane _ (by decide) l hl, o2.lane _ (by decide) l hl, L4 l hl, hdx, add_ofNat_zero] + have q5 : ∀ l < 2, s4.lane .xmm5 l = + s.mem.readW (coeffAddr fP (16 * m + 8) + BitVec.ofNat 64 (16 * l)) 128 := + fun l hl => by rw [l4, o3.lane _ (by decide) l hl, L5 l hl, o1.gpr, o1.mem, hdx, a1] + rw [WP.block_append_iff] + refine WP.mono (ybf_ok hbf hY c6 (x := fun l e => P[16 * m + 8 * l + e]!) + (y := fun l e => P[16 * m + 8 * l + 4 + e]!) (ζ := fun l _ => zetas (zi (2 * m + l))) + (fun l hl => by + rw [o6.lane _ (by decide) l hl, P5 l hl, perm20 _ _ hl] + rcases lane01 hl with rfl | rfl + · rw [ifp rfl, q4 0 (by decide)] + exact (dlanes_loadY hS j0 (by decide)).congr fun e _ => getP_congr P (by omega) + · rw [ifn (by decide), q5 0 (by decide)] + exact (dlanes_loadY hS j1 (by decide)).congr fun e _ => getP_congr P (by omega)) + (fun l hl => by + rw [P6 l hl, perm31 _ _ hl, o5.lane .xmm4 (by decide) 1 (by decide), o5.lane .xmm5 (by decide) 1 (by decide)] + rcases lane01 hl with rfl | rfl + · rw [ifp rfl, q4 1 (by decide)] + exact (dlanes_loadY hS j0 (by decide)).congr fun e _ => getP_congr P (by omega) + · rw [ifn (by decide), q5 1 (by decide)] + exact (dlanes_loadY hS j1 (by decide)).congr fun e _ => getP_congr P (by omega)) + (fun l hl => by + rw [(o5.trans o6).lane _ (by decide) l hl, l4] + rcases lane01 hl with rfl | rfl + · exact Z0.congr fun i hi => congrArg zetas (hsel i hi).1 + · exact Z1.congr fun i hi => congrArg zetas (hsel i hi).2) + (fun l hl => by + rw [(o5.trans o6).lane _ (by decide) l hl, (o5.trans o6).lane _ (by decide) l hl, l4, l4] + exact ZO l hl)) + fun s7 ⟨B7, o7⟩ => ?_ + -- the blocks back + rw [WP.block_append_iff] + refine WP.mono (yperm_ok s7) fun s8 ⟨P8, o8⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (yperm_ok s8) fun s9 ⟨P9, o9⟩ => ?_ + have o59 := (o5.trans o6).trans (o7.trans (o8.trans o9)) + have w0 : InRegions s9.wr (s9.gpr .rdx) 32 := by + rw [o59.wr, o59.gpr, g4.keep.2.2, g4.keep.gpr (by decide), o3.wr, o3.gpr, o12.wr, o12.gpr, hdx] + exact f_in32 hwf j0 + have w1 : InRegions s9.wr (s9.gpr .rdx + BitVec.ofNat 64 32) 32 := by + rw [o59.wr, o59.gpr, g4.keep.2.2, g4.keep.gpr (by decide), o3.wr, o3.gpr, o12.wr, o12.gpr, hdx, a1] + exact f_in32 hwf j1 + vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, State.setMem_ymm, + State.setMem_setMem, w0, w1, sx_ofNat (show 64 < 2 ^ 31 by decide)] + have g9 : s9.gpr = s4.gpr := o59.gpr + have m9 : s9.mem = s.mem := by rw [o59.mem, g4.mem, m3] + have dx9 : s9.gpr .rdx = coeffAddr fP (16 * m) := by + rw [g9, g4.keep.gpr (by decide), o3.gpr, o12.gpr, hdx] + rw [dx9, a1, m9] + have hR : ∀ i < 256, (layF blk F 4 zi (2 * (m + 1)))[i]! = if 16 * m ≤ i ∧ i < 16 * m + 16 then + (if i % (2 * 4) < 4 then (op P[i]! P[i + 4]! (zetas (zi (i / (2 * 4))))).1 + else (op P[i - 4]! P[i]! (zetas (zi (i / (2 * 4))))).2) else P[i]! := fun i hi => by + have hF : ∀ j, 16 * m ≤ j → j < 256 → P[j]! = F[j]! := fun j h1 h2 => by + rw [← hP, layF_get hblk F (by decide) zi (by omega) h2, ite_eq_right (by omega)] + rw [layF_get hblk F (by decide) zi (by omega) hi] + by_cases h1 : 16 * m ≤ i ∧ i < 16 * m + 16 + · rw [ite_eq_left (by omega), ite_eq_left h1] + by_cases h2 : i % (2 * 4) < 4 + · rw [ite_eq_left h2, ite_eq_left h2, hF _ h1.1 hi, hF _ (by omega) (by omega)] + · rw [ite_eq_right h2, ite_eq_right h2, hF _ (by omega) (by omega), hF _ h1.1 hi] + · rw [ite_eq_right h1, ← hP, layF_get hblk F (by decide) zi (by omega) hi] + by_cases h3 : i < 16 * m + · rw [ite_eq_left (show i < 2 * 4 * (2 * (m + 1)) by omega), + ite_eq_left (show i < 2 * 4 * (2 * m) by omega)] + · rw [ite_eq_right (show ¬ i < 2 * 4 * (2 * (m + 1)) by omega), + ite_eq_right (show ¬ i < 2 * 4 * (2 * m) by omega)] + refine ⟨?_, ?_, ?_, ?_, ?_, ⟨⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, ?_⟩⟩ + · refine polyIs_write2L (s := s9) hS j0 j1 (by omega) (fun l hl => ?_) (fun l hl => ?_) + (fun i hi h1 h2 => by rw [hR i hi, ite_eq_right (by omega)]) + · rw [o9.lane .xmm4 (by decide) l hl, P8 l hl, perm20 _ _ hl] + rcases lane01 hl with rfl | rfl + · rw [ifp rfl] + exact (B7 0 (by decide)).1.congr fun e he => by + rw [hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)] + exact congrArg Prod.fst (op_idx op P zi (by omega) (by omega) (by omega)) + · rw [ifn (by decide)] + exact (B7 0 (by decide)).2.congr fun e he => by + rw [hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)] + exact congrArg Prod.snd (op_idx op P zi (by omega) (by omega) (by omega)) + · rw [P9 l hl, perm31 _ _ hl, o8.lane .xmm0 (by decide) 1 (by decide), + o8.lane .xmm3 (by decide) 1 (by decide)] + rcases lane01 hl with rfl | rfl + · rw [ifp rfl] + exact (B7 1 (by decide)).1.congr fun e he => by + rw [hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)] + exact congrArg Prod.fst (op_idx op P zi (by omega) (by omega) (by omega)) + · rw [ifn (by decide)] + exact (B7 1 (by decide)).2.congr fun e he => by + rw [hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)] + exact congrArg Prod.snd (op_idx op P zi (by omega) (by omega) (by omega)) + · rw [show BitVec.signExtend 64 (64 : BitVec 32) = BitVec.ofNat 64 (4 * 16) by decide, coeffAddr_add, + Nat.mul_succ] + · rw [g9, h84, o3.gpr, o12.gpr] + · rw [g9, g4.keep.gpr (by decide), o3.gpr, o12.gpr] + · rw [g9, g4.keep.gpr (by decide), o3.gpr, o12.gpr] + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false, State.setMem_gpr] + rw [g9, g4.keep.gpr (by simp [hr]), o3.gpr, o12.gpr] + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd] + rw [o59.rd, g4.keep.2.1, o3.rd, o12.rd] + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr] + rw [o59.wr, g4.keep.2.2, o3.wr, o12.wr] + · exact frame_write2Y (Frame.refl _ _) j0 j1 _ _ + · exact ylanes_gpr (s := s9) (fun r l => by simp only [lane_setReg, lane_setFlags, State.setMem_lane]) + o59 (ylanes_gpr (s := s3) l4 (o12.trans o3) hc (by decide) (by decide)) (by decide) (by decide) + · exact o59.mxcsr.trans (g4.mxcsr.trans (o12.trans o3).mxcsr) + +theorem ylay4_ok {fP sP : Addr} (k : Nat) (o₀ o₁ : BitVec 8) (dz : BitVec 32) (zi kb : Nat → Nat) + (hkb0 : kb 0 = k) (hk : ∀ m < 16, kb m + 4 ≤ 256) + (hsel : ∀ m < 16, ∀ e < 4, kb m + sel o₀ e = zi (2 * m) ∧ kb m + sel o₁ e = zi (2 * m + 1)) + (hstep : ∀ m < 16, coeffAddr sP (kb m) + BitVec.signExtend 64 dz = coeffAddr sP (kb (m + 1))) + {F : Poly} {s : State} (hc : YConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) + (hd : (pR sP).Disjoint (pR fP)) : + WP isa (ylay4 bf k o₀ o₁ dz) s fun s' => PolyIs s'.mem fP (layF blk F 4 zi 32) ∧ BInvY fP s s' := by + have hk0 : k + 4 ≤ 256 := by have := hk 0 (by decide); omega + refine WP.seq (WP.mono (ypre21 k hk0 hdi hsi) fun w ⟨hdx, h8, og, hy⟩ => ?_) + have cw : YConsts w := ylanes_gpr (s := s) (og.lane hy) (YOnly.refl [] s) hc (by decide) (by decide) + refine WP.mono (wp_rcxLoopY (N := 16) (by decide) (by decide) + (fun i u => PolyIs u.mem fP (layF blk F 4 zi (2 * i)) ∧ u.gpr .rdx = coeffAddr fP (16 * i) ∧ + u.gpr .r8 = coeffAddr sP (kb i) ∧ BInvY fP w u) + (fun u ou hu _ => ⟨by rw [ou.mem, og.mem]; exact hS, + by rw [ou.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [ou.keep.gpr (by decide), h8, hkb0], ⟨ou.keep.mono (by simp), by rw [ou.mem]; exact Frame.refl _ _, + ylanes_gpr (s := w) (ou.lane hu) (YOnly.refl [] w) cw (by decide) (by decide), ou.mxcsr⟩⟩) + (fun i hi u ⟨hS', hdx', h8', hb'⟩ => ?_)) fun u ⟨hS', _, _, hb'⟩ => + ⟨hS', ⟨(og.keep.trans hb'.keep).mono (by simp), by rw [← og.mem]; exact hb'.frame, hb'.consts, + by rw [hb'.mxcsr, og.mxcsr]⟩⟩ + have hT' : Tab zmTab u.mem sP 256 := (by rw [og.mem]; exact hT : Tab zmTab w.mem sP 256).frame hb'.frame + (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide) + have hwf' : pR fP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hwf + have hw' : pR sP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hw + rw [show [Instr.vmovdquLoad .l256 .xmm4 (at_ .rdx 0), .vmovdquLoad .l256 .xmm5 (at_ .rdx 32)] ++ yzetaS o₀ o₁ ++ + [.alu .add .r8 (.imm dz), .vop (.vperm2i128 .xmm0 .xmm4 .xmm5 0x20), + .vop (.vperm2i128 .xmm1 .xmm4 .xmm5 0x31)] ++ toY bf ++ + [.vop (.vperm2i128 .xmm4 .xmm0 .xmm3 0x20), .vop (.vperm2i128 .xmm5 .xmm0 .xmm3 0x31), + .vmovdquStore .l256 (at_ .rdx 0) .xmm4, .vmovdquStore .l256 (at_ .rdx 32) .xmm5, + .alu .add .rdx (.imm 64)] ++ [.alu .sub .rcx (.imm 1)] = ybody4 bf o₀ o₁ dz by + simp [List.append_assoc]] + exact WP.mono (ystep4 hbf hY hblk hi o₀ o₁ dz zi (hk i hi) (hsel i hi) hb'.consts hdx' h8' hS' hT' hwf' hw') + fun u' ⟨hS'', hdx'', h8'', hcx, hzf, hb''⟩ => ⟨⟨hS'', hdx'', by rw [h8'', h8', hstep i hi], + hb'.trans hb''⟩, hcx, hzf⟩ + +end + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay21.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay21.lean new file mode 100644 index 000000000..f8bf6a6bd --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YLay21.lean @@ -0,0 +1,526 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YLay + +/-! +# ML-DSA on x86-64: the layers of the NTT and its inverse with `len` = 2 and 1 on AVX2 registers + +Untrusted: everything here is checked by Lean. Each iteration of these +layers loads sixteen coefficients, from `j`, into `ymm0` and `ymm1` (or +`ymm2`), and in each lane `l` runs `vlay2`'s or `vlay1`'s gathering, +butterflies and interleaving back (`Ntt.lean`) on the four coefficients +from `j + 4l` and the four from `j + 8 + 4l` (`core2_ok`, `core1_ok`), with +the zetas of their blocks in lane `l` of `ymm13` (`yzetaS_ok`, `yzeta8_ok`, +`yzeta8R_ok`); `ystep21` is an iteration for any such code, and `ylay2_ok` +and `ylay1_ok` the layers. +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (Keep XOnly XKeep YOnly ylanes yld_ok ifp ifn sel sel_lt add_ofNat_zero GOnly + addR_ok wp_rcxLoopY wp_cons_iff lane_setReg lane_setFlags State.setMem_ymm State.setMem_setMem q256lo q256hi + lo4 hi4) +open VG.Impl.MlKem.X86_64 (xb xmov toY rcxLoop) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt PolyIs zetas) + +/-! ## An iteration -/ + +/-- The body of the loop of the layers with `len` = 2 and 1. -/ +abbrev ybody21 (r2 : XReg) (zl core : List Instr) (dz : BitVec 32) : List Instr := + [.vmovdquLoad .l256 .xmm0 (at_ .rdx 0)] ++ ([.vmovdquLoad .l256 r2 (at_ .rdx 32)] ++ (zl ++ + ([.alu .add .r8 (.imm dz)] ++ (toY core ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1, .alu .add .rdx (.imm 64), + .alu .sub .rcx (.imm 1)])))) + +/-- An iteration of a layer with `len` = 2 or 1: the sixteen coefficients of +`G` from `j`, in lane `l` the four from `j + 4l` and the four from +`j + 8 + 4l`, become those of `R`, with the zetas `ζ l` that `zl` leaves in +the lanes of `ymm13`. -/ +theorem ystep21 {core zl : List Instr} {r2 : XReg} {zs : List XReg} {dz : BitVec 32} + (hY : laneSseBlock (toY core) = some core) (h0 : XReg.xmm0 ∉ zs) (h2 : r2 ∉ zs) + (h14 : XReg.xmm14 ∉ [XReg.xmm0] ++ [r2] ++ zs) (h15 : XReg.xmm15 ∉ [XReg.xmm0] ++ [r2] ++ zs) + (h20 : XReg.xmm0 ∉ [r2]) {fP : Addr} {j : Nat} (hj : j + 16 ≤ 256) {G R : Poly} + {ζ : Nat → Nat → Zq} {s : State} (hc : YConsts s) (hdx : s.gpr .rdx = coeffAddr fP j) + (hS : PolyIs s.mem fP G) (hw : pR fP ∈ s.wr) + (hz : ∀ s', XKeep s s' → + WP isa (.block zl) s' fun s'' => (∀ l < 2, ZLanes (s''.lane .xmm13 l) (ζ l) ∧ + ZOdd (s''.lane .xmm13 l) (s''.lane .xmm12 l)) ∧ YOnly zs s' s'') + (hcore : ∀ l < 2, ∀ t : State, VConsts t → DLanes (t.xmm .xmm0) (fun e => G[j + 4 * l + e]!) → + DLanes (t.xmm r2) (fun e => G[j + 8 + 4 * l + e]!) → ZLanes (t.xmm .xmm13) (ζ l) → + ZOdd (t.xmm .xmm13) (t.xmm .xmm12) → + WP isa (.block core) t fun t' => (DLanes (t'.xmm .xmm0) (fun e => R[j + 4 * l + e]!) ∧ + DLanes (t'.xmm .xmm1) (fun e => R[j + 8 + 4 * l + e]!)) ∧ + XOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm4] t t') + (hR : ∀ i < 256, i < j ∨ j + 16 ≤ i → R[i]! = G[i]!) : + WP isa (.block (ybody21 r2 zl core dz)) s fun s' => + PolyIs s'.mem fP R ∧ s'.gpr .rdx = coeffAddr fP (j + 16) ∧ + s'.gpr .r8 = s.gpr .r8 + BitVec.signExtend 64 dz ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ + s'.zf = some (s.gpr .rcx - 1 == 0) ∧ BInvY fP s s' := by + have j0 : j + 8 ≤ 256 := by omega + have j1 : j + 8 + 8 ≤ 256 := by omega + have a1 : coeffAddr fP j + BitVec.ofNat 64 32 = coeffAddr fP (j + 8) := coeffAddr_add _ _ 8 + have r0 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 0) 32 := by + rw [hdx, add_ofNat_zero]; exact f_in32 (List.mem_append_right s.rd hw) j0 + have r1 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 32) 32 := by + rw [hdx, a1]; exact f_in32 (List.mem_append_right s.rd hw) j1 + rw [ybody21, WP.block_append_iff] + refine WP.mono (yld_ok r0) fun s1 ⟨L0, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (yld_ok (d := r2) (by rw [o1.rd, o1.wr, o1.gpr]; exact r1)) fun s2 ⟨L2, o2⟩ => ?_ + have o12 := o1.trans o2 + rw [WP.block_append_iff] + refine WP.mono (hz s2 o12.toXKeep) fun s3 ⟨Z3, o3⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (addR_ok .r8 dz s3) fun s4 ⟨h84, g4, y4⟩ => ?_ + have l4 := g4.lane y4 + have c4 : YConsts s4 := ylanes_gpr (s := s3) l4 (o12.trans o3) hc h14 h15 + rw [WP.block_append_iff] + refine WP.mono (ylanes hY (P := fun l t => DLanes (t.xmm .xmm0) (fun e => R[j + 4 * l + e]!) ∧ + DLanes (t.xmm .xmm1) (fun e => R[j + 8 + 4 * l + e]!)) + fun l hl => hcore l hl _ (c4 l hl) + (by rw [State.proj_xmm, l4, o3.lane _ h0 l hl, o2.lane _ h20 l hl, L0 l hl, hdx, add_ofNat_zero] + exact dlanes_loadY hS j0 hl) + (by rw [State.proj_xmm, l4, o3.lane _ h2 l hl, L2 l hl, o1.gpr, o1.mem, hdx, a1] + exact dlanes_loadY hS j1 hl) + (by rw [State.proj_xmm, l4]; exact (Z3 l hl).1) + (by rw [State.proj_xmm, State.proj_xmm, l4, l4]; exact (Z3 l hl).2)) fun s5 ⟨C5, o5⟩ => ?_ + have m5 : s5.mem = s.mem := by rw [o5.mem, g4.mem, o3.mem, o12.mem] + have g5 : s5.gpr = s4.gpr := o5.gpr + have dx5 : s5.gpr .rdx = coeffAddr fP j := by rw [g5, g4.keep.gpr (by decide), o3.gpr, o12.gpr, hdx] + have k5 : s5.rd = s.rd ∧ s5.wr = s.wr := ⟨by rw [o5.rd, g4.keep.2.1, o3.rd, o12.rd], + by rw [o5.wr, g4.keep.2.2, o3.wr, o12.wr]⟩ + have w0 : InRegions s5.wr (s5.gpr .rdx) 32 := by rw [k5.2, dx5]; exact f_in32 hw j0 + have w1 : InRegions s5.wr (s5.gpr .rdx + BitVec.ofNat 64 32) 32 := by rw [k5.2, dx5, a1]; exact f_in32 hw j1 + vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, + State.setMem_ymm, State.setMem_setMem, w0, w1, sx_ofNat (show 64 < 2 ^ 31 by decide)] + rw [dx5, a1, m5] + refine ⟨?_, ?_, ?_, ?_, ?_, ⟨⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, ?_⟩⟩ + · exact polyIs_write2L (s := s5) hS j0 j1 (by omega) (fun l hl => (C5 l hl).1) + (fun l hl => (C5 l hl).2) fun i hi _ _ => hR i hi (by omega) + · rw [show BitVec.signExtend 64 (64 : BitVec 32) = BitVec.ofNat 64 (4 * 16) by decide, coeffAddr_add] + · rw [g5, h84, o3.gpr, o12.gpr] + · rw [g5, g4.keep.gpr (by decide), o3.gpr, o12.gpr] + · rw [g5, g4.keep.gpr (by decide), o3.gpr, o12.gpr] + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr, ite_false, State.setMem_gpr] + rw [g5, g4.keep.gpr (by simp [hr]), o3.gpr, o12.gpr] + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; exact k5.1 + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; exact k5.2 + · exact frame_write2Y (Frame.refl _ _) j0 j1 _ _ + · exact ylanes_gpr (s := s5) (fun r l => by simp only [lane_setReg, lane_setFlags, State.setMem_lane]) o5 c4 + (by decide) (by decide) + · exact o5.mxcsr.trans (g4.mxcsr.trans (o12.trans o3).mxcsr) + +/-! ## The gatherings, the butterflies and the interleavings back of a lane -/ + +theorem gath2_ok (t : State) : + WP isa (.block gath2) t fun t' => (t'.xmm .xmm0 = XBinOp.eval .punpcklqdq (t.xmm .xmm0) (t.xmm .xmm1) ∧ + t'.xmm .xmm1 = XBinOp.eval .punpckhqdq (t.xmm .xmm0) (t.xmm .xmm1)) ∧ XOnly [.xmm2, .xmm0, .xmm1] t t' := by + simp only [gath2, xmov, xb] + vrun [eval_movdqa] + exact ⟨by first | trivial | simp, by xonly⟩ + +theorem scat2_ok (t : State) : + WP isa (.block scat2) t fun t' => (t'.xmm .xmm0 = XBinOp.eval .punpcklqdq (t.xmm .xmm0) (t.xmm .xmm3) ∧ + t'.xmm .xmm1 = XBinOp.eval .punpckhqdq (t.xmm .xmm0) (t.xmm .xmm3)) ∧ XOnly [.xmm1, .xmm0] t t' := by + simp only [scat2, xmov, xb] + vrun [eval_movdqa] + exact ⟨by first | trivial | simp, by xonly⟩ + +theorem gath1_ok (t : State) : + WP isa (.block gath1) t fun t' => + (t'.xmm .xmm0 = XBinOp.eval .punpcklqdq (shufDwords (t.xmm .xmm0) 0xD8) (shufDwords (t.xmm .xmm2) 0xD8) ∧ + t'.xmm .xmm1 = XBinOp.eval .punpckhqdq (shufDwords (t.xmm .xmm0) 0xD8) (shufDwords (t.xmm .xmm2) 0xD8)) ∧ + XOnly [.xmm0, .xmm2, .xmm1] t t' := by + simp only [gath1, xmov, xb] + vrun [eval_movdqa] + exact ⟨by first | trivial | simp, by xonly⟩ + +theorem scat1_ok (t : State) : + WP isa (.block scat1) t fun t' => (t'.xmm .xmm0 = XBinOp.eval .punpckldq (t.xmm .xmm0) (t.xmm .xmm3) ∧ + t'.xmm .xmm1 = XBinOp.eval .punpckhdq (t.xmm .xmm0) (t.xmm .xmm3)) ∧ XOnly [.xmm1, .xmm0] t t' := by + simp only [scat1, xmov, xb] + vrun [eval_movdqa] + exact ⟨by first | trivial | simp, by xonly⟩ + +section +variable {bf : List Instr} {op : Zq → Zq → Zq → Zq × Zq} (hbf : VBflyOk bf op) +include hbf + +/-- `vlay2`'s work on the coefficients `A` of `xmm0` and `B` of `xmm1`: the +blocks `A` and `B` of `len = 2`, with the zetas `ζ` (the first two for `A`, +the last two for `B`). -/ +theorem core2_ok {t : State} (hc : VConsts t) {A B ζ : Nat → Zq} (hA : DLanes (t.xmm .xmm0) A) + (hB : DLanes (t.xmm .xmm1) B) (hz : ZLanes (t.xmm .xmm13) ζ) (ho : ZOdd (t.xmm .xmm13) (t.xmm .xmm12)) : + WP isa (.block (gath2 ++ bf ++ scat2)) t fun t' => + (DLanes (t'.xmm .xmm0) (fun e => if e < 2 then (op (A e) (A (2 + e)) (ζ e)).1 + else (op (A (e - 2)) (A e) (ζ (e - 2))).2) ∧ + DLanes (t'.xmm .xmm1) (fun e => if e < 2 then (op (B e) (B (2 + e)) (ζ (2 + e))).1 + else (op (B (e - 2)) (B e) (ζ e)).2)) ∧ XOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm4] t t' := by + rw [List.append_assoc, WP.block_append_iff] + refine WP.mono (gath2_ok t) fun t1 ⟨⟨e0, e1⟩, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (hbf _ (xonly_vconsts o1 hc (by decide) (by decide)) + (fun e => if e < 2 then A e else B (e - 2)) (fun e => if e < 2 then A (2 + e) else B e) ζ + (fun e he => by + dsimp only; rw [e0, dword_punpcklqdq _ _ he] + by_cases h : e < 2 <;> simp only [h, ite_true, ite_false] + · exact hA e he + · exact hB (e - 2) (by omega)) + (fun e he => by + dsimp only; rw [e1, dword_punpckhqdq _ _ he] + by_cases h : e < 2 <;> simp only [h, ite_true, ite_false] + · exact hA (2 + e) (by omega) + · exact hB e he) + (by rw [o1.xmm _ (by decide)]; exact hz) (by rw [o1.xmm _ (by decide), o1.xmm _ (by decide)]; exact ho)) + fun t2 ⟨X, Y, o2⟩ => ?_ + refine WP.mono (scat2_ok t2) fun t3 ⟨⟨f0, f1⟩, o3⟩ => ⟨⟨fun e he => ?_, fun e he => ?_⟩, ?_⟩ + · rw [f0, dword_punpcklqdq _ _ he] + split + · rw [X e he]; dsimp only + rw [ite_eq_left_of_eq_true _ _ (eq_true ‹e < 2›), ite_eq_left_of_eq_true _ _ (eq_true ‹e < 2›), + ite_eq_left_of_eq_true _ _ (eq_true ‹e < 2›)] + · rw [Y (e - 2) (by omega)]; dsimp only + rw [ite_eq_left_of_eq_true _ _ (eq_true (show e - 2 < 2 by omega)), + ite_eq_left_of_eq_true _ _ (eq_true (show e - 2 < 2 by omega)), + ite_eq_right_of_eq_false _ _ (eq_false ‹¬ e < 2›), show 2 + (e - 2) = e by omega] + · rw [f1, dword_punpckhqdq _ _ he] + split + · rw [X (2 + e) (by omega)]; dsimp only + rw [ite_eq_right_of_eq_false _ _ (eq_false (show ¬ 2 + e < 2 by omega)), + ite_eq_right_of_eq_false _ _ (eq_false (show ¬ 2 + e < 2 by omega)), + ite_eq_left_of_eq_true _ _ (eq_true ‹e < 2›), show 2 + e - 2 = e by omega] + · rw [Y e he]; dsimp only + rw [ite_eq_right_of_eq_false _ _ (eq_false ‹¬ e < 2›), ite_eq_right_of_eq_false _ _ (eq_false ‹¬ e < 2›), + ite_eq_right_of_eq_false _ _ (eq_false ‹¬ e < 2›)] + · exact ((o1.trans o2).trans o3).mono (by simp) + +/-- `vlay1`'s work on the coefficients `A` of `xmm0` and `B` of `xmm2`: the +blocks `A₀₁`, `A₂₃`, `B₀₁` and `B₂₃` of `len = 1`, with the zetas `ζ`. -/ +theorem core1_ok {t : State} (hc : VConsts t) {A B ζ : Nat → Zq} (hA : DLanes (t.xmm .xmm0) A) + (hB : DLanes (t.xmm .xmm2) B) (hz : ZLanes (t.xmm .xmm13) ζ) (ho : ZOdd (t.xmm .xmm13) (t.xmm .xmm12)) : + WP isa (.block (gath1 ++ bf ++ scat1)) t fun t' => + (DLanes (t'.xmm .xmm0) (fun e => if e % 2 = 0 then (op (A e) (A (e + 1)) (ζ (e / 2))).1 + else (op (A (e - 1)) (A e) (ζ (e / 2))).2) ∧ + DLanes (t'.xmm .xmm1) (fun e => if e % 2 = 0 then (op (B e) (B (e + 1)) (ζ (2 + e / 2))).1 + else (op (B (e - 1)) (B e) (ζ (2 + e / 2))).2)) ∧ XOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm4] t t' := by + rw [List.append_assoc, WP.block_append_iff] + refine WP.mono (gath1_ok t) fun t1 ⟨⟨e0, e1⟩, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (hbf _ (xonly_vconsts o1 hc (by decide) (by decide)) + (fun e => if e < 2 then A (2 * e) else B (2 * (e - 2))) (fun e => if e < 2 then A (2 * e + 1) else B (2 * (e - 2) + 1)) + ζ + (fun e he => by + dsimp only; rw [e0, dword_punpcklqdq _ _ he] + by_cases h : e < 2 <;> simp only [h, ite_true, ite_false] + · rw [dword_d8 _ he, ite_eq_left h]; exact hA _ (by omega) + · rw [dword_d8 _ (by omega), ite_eq_left (by omega)]; exact hB _ (by omega)) + (fun e he => by + dsimp only; rw [e1, dword_punpckhqdq _ _ he] + by_cases h : e < 2 <;> simp only [h, ite_true, ite_false] + · rw [dword_d8 _ (by omega), ite_eq_right (by omega), show 2 * (2 + e - 2) + 1 = 2 * e + 1 by omega] + exact hA _ (by omega) + · rw [dword_d8 _ he, ite_eq_right h]; exact hB _ (by omega)) + (by rw [o1.xmm _ (by decide)]; exact hz) (by rw [o1.xmm _ (by decide), o1.xmm _ (by decide)]; exact ho)) + fun t2 ⟨X, Y, o2⟩ => ?_ + refine WP.mono (scat1_ok t2) fun t3 ⟨⟨f0, f1⟩, o3⟩ => ⟨⟨fun e he => ?_, fun e he => ?_⟩, ?_⟩ + · rw [f0, dword_punpckldq' _ _ he] + split + · rw [X (e / 2) (by omega)]; dsimp only + rw [ite_eq_left (by omega), ite_eq_left (by omega), ite_eq_left ‹e % 2 = 0›, + show 2 * (e / 2) = e by omega] + · rw [Y (e / 2) (by omega)]; dsimp only + rw [ite_eq_left (by omega), ite_eq_left (by omega), ite_eq_right ‹¬ e % 2 = 0›, + show 2 * (e / 2) = e - 1 by omega, show e - 1 + 1 = e by omega] + · rw [f1, dword_punpckhdq' _ _ he] + split + · rw [X (2 + e / 2) (by omega)]; dsimp only + rw [ite_eq_right (by omega), ite_eq_right (by omega), ite_eq_left ‹e % 2 = 0›, + show 2 * (2 + e / 2 - 2) = e by omega] + · rw [Y (2 + e / 2) (by omega)]; dsimp only + rw [ite_eq_right (by omega), ite_eq_right (by omega), ite_eq_right ‹¬ e % 2 = 0›, + show 2 * (2 + e / 2 - 2) = e - 1 by omega, show e - 1 + 1 = e by omega] + · exact ((o1.trans o2).trans o3).mono (by simp) + +end + +/-! ## The zetas of the layer with `len = 1` -/ + +/-- A doubleword of four of the zetas, at index `i` of the table. -/ +theorem dword_tab {m : Mem} {zP : Addr} (ht : Tab zmTab m zP 256) {j e i : Nat} (he : e < 4) (hi : j + e = i) + (hi' : i < 256) : (dword (m.readW (coeffAddr zP j) 128) e).toNat = (zetas i).val * 2 ^ 32 % q := by + subst hi; rw [dword_readW _ _ he, coeffAddr_add]; exact tab_zeta ht hi' + +/-- `vpermq` with `0x27`: lane 0 is `punpckhqdq` of the upper and the lower lane, lane 1 their `punpcklqdq`. -/ +theorem perm27 (a b : BitVec 128) : + permQwords (b ++ a) 0x27 = qword256 (b ++ a) 0 ++ qword256 (b ++ a) (2 + 0) ++ + qword256 (b ++ a) 1 ++ qword256 (b ++ a) (2 + 1) := rfl + +theorem perm27_lo (a b : BitVec 128) : + (permQwords (b ++ a) 0x27).extractLsb' 0 128 = XBinOp.eval .punpckhqdq b a := by + rw [perm27, q256hi, q256hi, q256lo _ _ (by decide), q256lo _ _ (by decide), lo4]; rfl + +theorem perm27_hi (a b : BitVec 128) : + (permQwords (b ++ a) 0x27).extractLsb' 128 128 = XBinOp.eval .punpcklqdq b a := by + rw [perm27, q256hi, q256hi, q256lo _ _ (by decide), q256lo _ _ (by decide), hi4]; rfl + +/-- `vpermq d, r, 0x27`. -/ +theorem ypermq27_ok {d r : XReg} (s : State) : + WP isa (.block [.vop (.vpermq d r 0x27)]) s fun s' => + s'.lane d 0 = XBinOp.eval .punpckhqdq (s.lane r 1) (s.lane r 0) ∧ + s'.lane d 1 = XBinOp.eval .punpcklqdq (s.lane r 1) (s.lane r 0) ∧ YOnly [d] s s' := by + apply WP.of_runBlock + simp only [runBlock_cons, runStep_some, runBlock_nil, exec, VOp.exec, Option.some.injEq, exists_eq_left'] + refine ⟨?_, ?_, ⟨⟨rfl, rfl, rfl, rfl, rfl⟩, fun r' hr l hl => ?_⟩⟩ + · rw [State.lane_setV256, ifp rfl, ifp rfl, State.ymm_eq, perm27_lo] + · rw [State.lane_setV256, ifp rfl, ifn (by decide), State.ymm_eq, perm27_hi] + · rw [State.lane_setV256, ifn (by simpa using hr)] + +/-- The eight zetas at index `k` of the table, in the order of the blocks of +`ylay1` for `NTT`: in lane `l`, zetas `k + 2l`, `k + 2l + 1`, `k + 2l + 4` +and `k + 2l + 5`. -/ +theorem yzeta8_ok {zP : Addr} {k : Nat} (hk : k + 8 ≤ 256) {s : State} (h8 : s.gpr .r8 = coeffAddr zP k) + (hin : InRegions (s.rd ++ s.wr) (coeffAddr zP k) 32) (ht : Tab zmTab s.mem zP 256) : + WP isa (.block yzeta8) s fun s' => + (∀ l < 2, ZLanes (s'.lane .xmm13 l) (fun e => zetas (k + (2 * l + e + 2 * (e / 2)))) ∧ + ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧ YOnly [.xmm13, .xmm12] s s' := by + rw [yzeta8, WP.block_append_iff, wp_cons_iff] + refine WP.mono (yld_ok (by rw [h8, add_ofNat_zero]; exact hin)) fun s1 ⟨L1, o1⟩ => ?_ + refine WP.mono (VG.Proof.MlKem.X86_64.ypermq_ok s1) fun s2 ⟨e0, e1, o2⟩ => ?_ + refine WP.mono (yF5_ok s2) fun s3 ⟨f3, o3⟩ => ⟨fun l hl => ?_, ((o1.trans o2).trans o3).mono (by simp)⟩ + rw [f3 l hl, o3.lane _ (by decide) l hl] + refine ⟨fun e he => ?_, zodd_F5 _⟩ + have a : ∀ l < 2, s1.lane .xmm13 l = s.mem.readW (coeffAddr zP (k + 4 * l)) 128 := fun l hl => by + rw [L1 l hl, h8, add_ofNat_zero, lane_load] + rcases lane01 hl with rfl | rfl + · rw [e0, dword_punpcklqdq _ _ he, a 0 (by decide), a 1 (by decide)] + split + · exact dword_tab ht he (by omega) (by omega) + · exact dword_tab ht (by omega) (by omega) (by omega) + · rw [e1, dword_punpckhqdq _ _ he, a 0 (by decide), a 1 (by decide)] + split + · exact dword_tab ht (by omega) (by omega) (by omega) + · exact dword_tab ht he (by omega) (by omega) + +theorem dword_b1 (x : BitVec 128) {e : Nat} (he : e < 4) : + dword (shufDwords x 0xB1) e = dword x (if e % 2 = 0 then e + 1 else e - 1) := by + rw [dword_shufDwords _ _ he] + rcases cases4 he with rfl | rfl | rfl | rfl <;> rfl + +theorem zsseR_ok (t : State) : + WP isa (.block [.xop (.pshufd .xmm13 .xmm13 0xB1), .xop (.pshufd .xmm12 .xmm13 0xF5)]) t fun t' => + (t'.xmm .xmm13 = shufDwords (t.xmm .xmm13) 0xB1 ∧ + t'.xmm .xmm12 = shufDwords (shufDwords (t.xmm .xmm13) 0xB1) 0xF5) ∧ XOnly [.xmm13, .xmm12] t t' := by + vrun + exact ⟨by first | trivial | simp, by xonly⟩ + +/-- The eight zetas at index `k` of the table, in the order of the blocks of +`ylay1` for `NTT⁻¹`, which take them in decreasing order: in lane `l`, zetas +`k + 7 - 2l`, `k + 6 - 2l`, `k + 3 - 2l` and `k + 2 - 2l`. -/ +theorem yzeta8R_ok {zP : Addr} {k : Nat} (hk : k + 8 ≤ 256) {s : State} (h8 : s.gpr .r8 = coeffAddr zP k) + (hin : InRegions (s.rd ++ s.wr) (coeffAddr zP k) 32) (ht : Tab zmTab s.mem zP 256) : + WP isa (.block yzeta8R) s fun s' => + (∀ l < 2, ZLanes (s'.lane .xmm13 l) (fun e => zetas (k + 7 - (2 * l + e + 2 * (e / 2)))) ∧ + ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧ YOnly [.xmm13, .xmm12] s s' := by + rw [yzeta8R, WP.block_append_iff, wp_cons_iff] + refine WP.mono (yld_ok (by rw [h8, add_ofNat_zero]; exact hin)) fun s1 ⟨L1, o1⟩ => ?_ + refine WP.mono (ypermq27_ok s1) fun s2 ⟨e0, e1, o2⟩ => ?_ + refine WP.mono (ylanes (by decide) (P := fun l t => t.xmm .xmm13 = shufDwords (s2.lane .xmm13 l) 0xB1 ∧ + t.xmm .xmm12 = shufDwords (shufDwords (s2.lane .xmm13 l) 0xB1) 0xF5) + fun l _ => zsseR_ok (s2.proj l)) fun s3 ⟨l3, o3⟩ => ⟨fun l hl => ?_, ((o1.trans o2).trans o3).mono (by simp)⟩ + have e13 : s3.lane .xmm13 l = _ := (l3 l hl).1 + have e12 : s3.lane .xmm12 l = _ := (l3 l hl).2 + rw [e12, e13] + refine ⟨fun e he => ?_, zodd_F5 _⟩ + have a : ∀ l < 2, s1.lane .xmm13 l = s.mem.readW (coeffAddr zP (k + 4 * l)) 128 := fun l hl => by + rw [L1 l hl, h8, add_ofNat_zero, lane_load] + have he' : (if e % 2 = 0 then e + 1 else e - 1) < 4 := by split <;> omega + rw [dword_b1 _ he] + generalize hf : (if e % 2 = 0 then e + 1 else e - 1) = f at he' + have hf' : f = if e % 2 = 0 then e + 1 else e - 1 := hf.symm + rcases lane01 hl with rfl | rfl + · rw [e0, dword_punpckhqdq _ _ he', a 0 (by decide), a 1 (by decide)] + split + · exact dword_tab ht (by omega) (by split at hf' <;> omega) (by omega) + · exact dword_tab ht he' (by split at hf' <;> omega) (by omega) + · rw [e1, dword_punpcklqdq _ _ he', a 0 (by decide), a 1 (by decide)] + split + · exact dword_tab ht he' (by split at hf' <;> omega) (by omega) + · exact dword_tab ht (by omega) (by split at hf' <;> omega) (by omega) + +/-! ## The layers -/ + +section +variable {op : Zq → Zq → Zq → Zq × Zq} {blk : Poly → Nat → Nat → Nat → Nat → Poly} (hblk : BlkOk blk op) +include hblk + +/-- The sixteen coefficients from `16i` after the next blocks of the layer +with `len = 2`. -/ +theorem layF2_next (F : Poly) (zi : Nat → Nat) {i : Nat} (hi : i < 16) {x : Nat} (hx : x < 256) : + (layF blk F 2 zi (4 * (i + 1)))[x]! = if 16 * i ≤ x ∧ x < 16 * i + 16 then + (if x % (2 * 2) < 2 then + (op (layF blk F 2 zi (4 * i))[x]! (layF blk F 2 zi (4 * i))[x + 2]! (zetas (zi (x / (2 * 2))))).1 + else (op (layF blk F 2 zi (4 * i))[x - 2]! (layF blk F 2 zi (4 * i))[x]! (zetas (zi (x / (2 * 2))))).2) + else (layF blk F 2 zi (4 * i))[x]! := by + have hF : ∀ y, 16 * i ≤ y → y < 256 → (layF blk F 2 zi (4 * i))[y]! = F[y]! := fun y h1 h2 => by + rw [layF_get hblk F (by decide) zi (by omega) h2, ite_eq_right (by omega)] + rw [layF_get hblk F (by decide) zi (by omega) hx] + by_cases h1 : 16 * i ≤ x ∧ x < 16 * i + 16 + · rw [ite_eq_left (by omega), ite_eq_left h1] + by_cases h2 : x % (2 * 2) < 2 + · rw [ite_eq_left h2, ite_eq_left h2, hF _ h1.1 hx, hF _ (by omega) (by omega)] + · rw [ite_eq_right h2, ite_eq_right h2, hF _ (by omega) (by omega), hF _ h1.1 hx] + · rw [ite_eq_right h1, layF_get hblk F (by decide) zi (by omega) hx] + by_cases h3 : x < 16 * i + · rw [ite_eq_left (show x < 2 * 2 * (4 * (i + 1)) by omega), ite_eq_left (show x < 2 * 2 * (4 * i) by omega)] + · rw [ite_eq_right (show ¬ x < 2 * 2 * (4 * (i + 1)) by omega), + ite_eq_right (show ¬ x < 2 * 2 * (4 * i) by omega)] + +/-- The sixteen coefficients from `16i` after the next blocks of the layer +with `len = 1`. -/ +theorem layF1_next (F : Poly) (zi : Nat → Nat) {i : Nat} (hi : i < 16) {x : Nat} (hx : x < 256) : + (layF blk F 1 zi (8 * (i + 1)))[x]! = if 16 * i ≤ x ∧ x < 16 * i + 16 then + (if x % (2 * 1) < 1 then + (op (layF blk F 1 zi (8 * i))[x]! (layF blk F 1 zi (8 * i))[x + 1]! (zetas (zi (x / (2 * 1))))).1 + else (op (layF blk F 1 zi (8 * i))[x - 1]! (layF blk F 1 zi (8 * i))[x]! (zetas (zi (x / (2 * 1))))).2) + else (layF blk F 1 zi (8 * i))[x]! := by + have hF : ∀ y, 16 * i ≤ y → y < 256 → (layF blk F 1 zi (8 * i))[y]! = F[y]! := fun y h1 h2 => by + rw [layF_get hblk F (by decide) zi (by omega) h2, ite_eq_right (by omega)] + rw [layF_get hblk F (by decide) zi (by omega) hx] + by_cases h1 : 16 * i ≤ x ∧ x < 16 * i + 16 + · rw [ite_eq_left (by omega), ite_eq_left h1] + by_cases h2 : x % (2 * 1) < 1 + · rw [ite_eq_left h2, ite_eq_left h2, hF _ h1.1 hx, hF _ (by omega) (by omega)] + · rw [ite_eq_right h2, ite_eq_right h2, hF _ (by omega) (by omega), hF _ h1.1 hx] + · rw [ite_eq_right h1, layF_get hblk F (by decide) zi (by omega) hx] + by_cases h3 : x < 16 * i + · rw [ite_eq_left (show x < 2 * 1 * (8 * (i + 1)) by omega), ite_eq_left (show x < 2 * 1 * (8 * i) by omega)] + · rw [ite_eq_right (show ¬ x < 2 * 1 * (8 * (i + 1)) by omega), + ite_eq_right (show ¬ x < 2 * 1 * (8 * i) by omega)] + +variable {bf : List Instr} (hbf : VBflyOk bf op) +include hbf + +theorem ylay2_ok (hY : laneSseBlock (toY (gath2 ++ bf ++ scat2)) = some (gath2 ++ bf ++ scat2)) {fP sP : Addr} + (k : Nat) (o₀ o₁ : BitVec 8) (dz : BitVec 32) (zi kb : Nat → Nat) (hkb0 : kb 0 = k) + (hk : ∀ i < 16, kb i + 4 ≤ 256) + (hsel : ∀ i < 16, ∀ e < 4, kb i + sel o₀ e = zi (4 * i + 2 * (e / 2)) ∧ + kb i + sel o₁ e = zi (4 * i + 1 + 2 * (e / 2))) + (hstep : ∀ i < 16, coeffAddr sP (kb i) + BitVec.signExtend 64 dz = coeffAddr sP (kb (i + 1))) + {F : Poly} {s : State} (hc : YConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) + (hd : (pR sP).Disjoint (pR fP)) : + WP isa (ylay2 bf k o₀ o₁ dz) s fun s' => PolyIs s'.mem fP (layF blk F 2 zi 64) ∧ BInvY fP s s' := by + have hk0 : k + 4 ≤ 256 := by have := hk 0 (by decide); omega + refine WP.seq (WP.mono (ypre21 k hk0 hdi hsi) fun w ⟨hdx, h8, og, hy⟩ => ?_) + have cw : YConsts w := ylanes_gpr (s := s) (og.lane hy) (YOnly.refl [] s) hc (by decide) (by decide) + refine WP.mono (wp_rcxLoopY (N := 16) (by decide) (by decide) + (fun i u => PolyIs u.mem fP (layF blk F 2 zi (4 * i)) ∧ u.gpr .rdx = coeffAddr fP (16 * i) ∧ + u.gpr .r8 = coeffAddr sP (kb i) ∧ BInvY fP w u) + (fun u ou hu _ => ⟨by rw [ou.mem, og.mem]; exact hS, + by rw [ou.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [ou.keep.gpr (by decide), h8, hkb0], ⟨ou.keep.mono (by simp), by rw [ou.mem]; exact Frame.refl _ _, + ylanes_gpr (s := w) (ou.lane hu) (YOnly.refl [] w) cw (by decide) (by decide), ou.mxcsr⟩⟩) + (fun i hi u ⟨hS', hdx', h8', hb'⟩ => ?_)) fun u ⟨hS', _, _, hb'⟩ => + ⟨hS', ⟨(og.keep.trans hb'.keep).mono (by simp), by rw [← og.mem]; exact hb'.frame, hb'.consts, + by rw [hb'.mxcsr, og.mxcsr]⟩⟩ + have hT' : Tab zmTab u.mem sP 256 := (by rw [og.mem]; exact hT : Tab zmTab w.mem sP 256).frame hb'.frame + (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide) + have hwf' : pR fP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hwf + have hw' : pR sP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hw + have hk0' : ∀ j < 4, kb i + sel o₀ j < 256 := fun j _ => by have := sel_lt o₀ j; have := hk i hi; omega + have hk1' : ∀ j < 4, kb i + sel o₁ j < 256 := fun j _ => by have := sel_lt o₁ j; have := hk i hi; omega + rw [show [Instr.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm1 (at_ .rdx 32)] ++ yzetaS o₀ o₁ ++ + [.alu .add .r8 (.imm dz)] ++ toY (gath2 ++ bf ++ scat2) ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1, + .alu .add .rdx (.imm 64)] ++ [.alu .sub .rcx (.imm 1)] = + ybody21 .xmm1 (yzetaS o₀ o₁) (gath2 ++ bf ++ scat2) dz by simp [List.append_assoc]] + have hR := fun x hx => layF2_next hblk F zi hi (x := x) hx + refine WP.mono (ystep21 hY (zs := [.xmm13, .xmm2, .xmm12]) (by decide) (by decide) (by decide) (by decide) + (by decide) (j := 16 * i) (by omega) (R := layF blk F 2 zi (4 * (i + 1))) + (ζ := fun l e => zetas (zi (4 * i + l + 2 * (e / 2)))) hb'.consts hdx' hS' hwf' (fun s' k' => ?_) + (fun l hl t ht hA hB hz ho => ?_) (fun x hx h => by rw [hR x hx, ite_eq_right (by omega)])) + fun u' ⟨hS'', hdx'', h8'', hcx, hzf, hb''⟩ => ⟨⟨hS'', by rw [hdx'', Nat.mul_succ], + by rw [h8'', h8', hstep i hi], hb'.trans hb''⟩, hcx, hzf⟩ + · -- the zetas + refine WP.mono (yzetaS_ok o₀ o₁ (zP := sP) (k := kb i) hk0' hk1' (by rw [k'.gpr, h8']) + (by rw [k'.rd, k'.wr]; exact tab_in (List.mem_append_right _ hw') (hk i hi)) (by rw [k'.mem]; exact hT')) + fun s'' ⟨Z0, Z1, ZO, o⟩ => ⟨fun l hl => ⟨?_, ZO l hl⟩, o⟩ + rcases lane01 hl with rfl | rfl + · exact Z0.congr fun e he => congrArg zetas ((hsel i hi e he).1.trans (congrArg zi (by omega))) + · exact Z1.congr fun e he => congrArg zetas ((hsel i hi e he).2.trans (congrArg zi (by omega))) + · -- the blocks of a lane + refine WP.mono (core2_ok hbf ht hA hB hz ho) fun t' ⟨⟨a, b⟩, o⟩ => + ⟨⟨a.congr fun e he => ?_, b.congr fun e he => ?_⟩, o⟩ + · by_cases h : e < 2 + · rw [ite_eq_left h, hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)] + exact congrArg Prod.fst (op_idx op _ zi (by omega) (by omega) (by omega)) + · rw [ite_eq_right h, hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)] + exact congrArg Prod.snd (op_idx op _ zi (by omega) (by omega) (by omega)) + · by_cases h : e < 2 + · rw [ite_eq_left h, hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)] + exact congrArg Prod.fst (op_idx op _ zi (by omega) (by omega) (by omega)) + · rw [ite_eq_right h, hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)] + exact congrArg Prod.snd (op_idx op _ zi (by omega) (by omega) (by omega)) + +theorem ylay1_ok (hY : laneSseBlock (toY (gath1 ++ bf ++ scat1)) = some (gath1 ++ bf ++ scat1)) {fP sP : Addr} + (k : Nat) (zl : List Instr) (dz : BitVec 32) (zi kb : Nat → Nat) (hkb0 : kb 0 = k) + (hk : ∀ i < 16, kb i + 8 ≤ 256) + (hzl : ∀ i < 16, ∀ s : State, s.gpr .r8 = coeffAddr sP (kb i) → + InRegions (s.rd ++ s.wr) (coeffAddr sP (kb i)) 32 → Tab zmTab s.mem sP 256 → + WP isa (.block zl) s fun s' => (∀ l < 2, ZLanes (s'.lane .xmm13 l) + (fun e => zetas (zi (8 * i + 2 * l + e + 2 * (e / 2)))) ∧ ZOdd (s'.lane .xmm13 l) (s'.lane .xmm12 l)) ∧ + YOnly [.xmm13, .xmm12] s s') + (hstep : ∀ i < 16, coeffAddr sP (kb i) + BitVec.signExtend 64 dz = coeffAddr sP (kb (i + 1))) + {F : Poly} {s : State} (hc : YConsts s) (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) + (hd : (pR sP).Disjoint (pR fP)) : + WP isa (ylay1 bf k zl dz) s fun s' => PolyIs s'.mem fP (layF blk F 1 zi 128) ∧ BInvY fP s s' := by + have hk0 : k + 4 ≤ 256 := by have := hk 0 (by decide); omega + refine WP.seq (WP.mono (ypre21 k hk0 hdi hsi) fun w ⟨hdx, h8, og, hy⟩ => ?_) + have cw : YConsts w := ylanes_gpr (s := s) (og.lane hy) (YOnly.refl [] s) hc (by decide) (by decide) + refine WP.mono (wp_rcxLoopY (N := 16) (by decide) (by decide) + (fun i u => PolyIs u.mem fP (layF blk F 1 zi (8 * i)) ∧ u.gpr .rdx = coeffAddr fP (16 * i) ∧ + u.gpr .r8 = coeffAddr sP (kb i) ∧ BInvY fP w u) + (fun u ou hu _ => ⟨by rw [ou.mem, og.mem]; exact hS, + by rw [ou.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [ou.keep.gpr (by decide), h8, hkb0], ⟨ou.keep.mono (by simp), by rw [ou.mem]; exact Frame.refl _ _, + ylanes_gpr (s := w) (ou.lane hu) (YOnly.refl [] w) cw (by decide) (by decide), ou.mxcsr⟩⟩) + (fun i hi u ⟨hS', hdx', h8', hb'⟩ => ?_)) fun u ⟨hS', _, _, hb'⟩ => + ⟨hS', ⟨(og.keep.trans hb'.keep).mono (by simp), by rw [← og.mem]; exact hb'.frame, hb'.consts, + by rw [hb'.mxcsr, og.mxcsr]⟩⟩ + have hT' : Tab zmTab u.mem sP 256 := (by rw [og.mem]; exact hT : Tab zmTab w.mem sP 256).frame hb'.frame + (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) (by decide) + have hwf' : pR fP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hwf + have hw' : pR sP ∈ u.wr := by rw [hb'.keep.2.2, og.keep.2.2]; exact hw + rw [show [Instr.vmovdquLoad .l256 .xmm0 (at_ .rdx 0), .vmovdquLoad .l256 .xmm2 (at_ .rdx 32)] ++ zl ++ + [.alu .add .r8 (.imm dz)] ++ toY (gath1 ++ bf ++ scat1) ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm0, .vmovdquStore .l256 (at_ .rdx 32) .xmm1, + .alu .add .rdx (.imm 64)] ++ [.alu .sub .rcx (.imm 1)] = + ybody21 .xmm2 zl (gath1 ++ bf ++ scat1) dz by simp [List.append_assoc]] + have hR := fun x hx => layF1_next hblk F zi hi (x := x) hx + refine WP.mono (ystep21 hY (zs := [.xmm13, .xmm12]) (by decide) (by decide) (by decide) (by decide) + (by decide) (j := 16 * i) (by omega) (R := layF blk F 1 zi (8 * (i + 1))) + (ζ := fun l e => zetas (zi (8 * i + 2 * l + e + 2 * (e / 2)))) hb'.consts hdx' hS' hwf' + (fun s' k' => hzl i hi s' (by rw [k'.gpr, h8']) + (by rw [k'.rd, k'.wr]; exact f_in32 (List.mem_append_right _ hw') (hk i hi)) (by rw [k'.mem]; exact hT')) + (fun l hl t ht hA hB hz ho => ?_) (fun x hx h => by rw [hR x hx, ite_eq_right (by omega)])) + fun u' ⟨hS'', hdx'', h8'', hcx, hzf, hb''⟩ => ⟨⟨hS'', by rw [hdx'', Nat.mul_succ], + by rw [h8'', h8', hstep i hi], hb'.trans hb''⟩, hcx, hzf⟩ + -- the blocks of a lane + refine WP.mono (core1_ok hbf ht hA hB hz ho) fun t' ⟨⟨a, b⟩, o⟩ => + ⟨⟨a.congr fun e he => ?_, b.congr fun e he => ?_⟩, o⟩ + · by_cases h : e % 2 = 0 + · rw [ite_eq_left h, hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)] + exact congrArg Prod.fst (op_idx op _ zi (by omega) (by omega) (by omega)) + · rw [ite_eq_right h, hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)] + exact congrArg Prod.snd (op_idx op _ zi (by omega) (by omega) (by omega)) + · by_cases h : e % 2 = 0 + · rw [ite_eq_left h, hR _ (by omega), ite_eq_left (by omega), ite_eq_left (by omega)] + exact congrArg Prod.fst (op_idx op _ zi (by omega) (by omega) (by omega)) + · rw [ite_eq_right h, hR _ (by omega), ite_eq_left (by omega), ite_eq_right (by omega)] + exact congrArg Prod.snd (op_idx op _ zi (by omega) (by omega) (by omega)) + +end + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMul.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMul.lean new file mode 100644 index 000000000..32c854216 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMul.lean @@ -0,0 +1,392 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YBase +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul + +/-! +# ML-DSA on x86-64: `vg_mldsa_multiply_ntt_avx2` and `vg_mldsa_multiply_add_ntt_avx2` + +Untrusted: everything here is checked by Lean. As `vg_mldsa_multiply_ntt` +and `vg_mldsa_multiply_add_ntt` (`Mul.lean`) on eight coefficients at a +time: each iteration of the loop loads eight coefficients of `f`, `g` and +`h` and, in each lane, does what the SSE2 code's `mulCore` (`mulAddCore`) +does (`ylanes`), on the coefficients `8i + 4l` to `8i + 4l + 3` of lane +`l`, and stores the eight results (`YMul.step`); the loop stores the first +248 (`YMul.loop_ok`), and the last eight, from the coefficients of `h` in +`ymm6` (`YMul.last`), are stored after MXCSR is loaded back +(`YMul.fn_ok`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes yld_ok yconst_ok WP.keep writesOnly gprPreserved_of ifp ifn + ptr_step GOnly wp_rcxLoopY add_ofNat_zero lane_setReg lane_setFlags sx32 State.setMem_ymm xmm_setXmm) +open VG.Impl.MlKem.X86_64 (xb xmov toY yconst rcxLoop) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced) + +theorem lane_mulCore : laneSseBlock (toY mulCore) = some mulCore := by decide +kernel +theorem lane_mulAddCore : laneSseBlock (toY mulAddCore) = some mulAddCore := by decide +kernel + +/-- The constants, and `2⁶⁴ mod q` in both lanes of `ymm11`. -/ +theorem ymulPro_ok (s : State) : + WP isa (.block ymulPro) s fun s' => YConsts s' ∧ (∀ l < 2, s'.lane .xmm11 l = r2V) ∧ + (∀ l < 2, s'.lane .xmm6 l = s.lane .xmm6 l) ∧ Keep [.rax] s s' ∧ s'.mem = s.mem := by + rw [ymulPro, WP.block_append_iff] + refine WP.mono (yconsts_ok s) fun s1 ⟨c1, k1, m1, _, o1⟩ => + WP.mono (yconst_ok .xmm11 _ s1) fun s2 ⟨l2, k2, m2, _, o2⟩ => + ⟨fun l hl => ⟨?_, ?_⟩, fun l hl => by rw [l2 l hl]; decide, + fun l hl => by rw [o2 _ (by decide) l hl, o1 _ (by decide) (by decide) l hl], + (k1.trans k2).mono (by simp), m2.trans m1⟩ + · rw [State.proj_xmm, o2 _ (by decide) l hl]; exact (c1 l hl).q + · rw [State.proj_xmm, o2 _ (by decide) l hl]; exact (c1 l hl).qinv + +namespace YMul + +/-- After `i` iterations: the first `8i` coefficients of `h` are `R`'s, the +others as they were in `s₀`. -/ +structure Inv (h f g : Addr) (s₀ : State) (R : Poly) (i : Nat) (s : State) : Prop where + rdi : s.gpr .rdi = h + BitVec.ofNat 64 (32 * i) + rsi : s.gpr .rsi = f + BitVec.ofNat 64 (32 * i) + rdx : s.gpr .rdx = g + BitVec.ofNat 64 (32 * i) + rd : s.rd = s₀.rd + wr : s.wr = s₀.wr + c : YConsts s + r2 : ∀ l < 2, s.lane .xmm11 l = r2V + x6 : ∀ l < 2, s.lane .xmm6 l = s₀.lane .xmm6 l + frame : Frame [pR h] s₀.mem s.mem + done : ∀ k < 8 * i, (coeffAt s.mem h k).toNat = (R[k]!).val + rest : ∀ k < 256, 8 * i ≤ k → coeffAt s.mem h k = coeffAt s₀.mem h k + +section +variable {h f g : Addr} {s₀ : State} (hwh : pR h ∈ s₀.wr) (hrf : pR f ∈ s₀.rd ++ s₀.wr) + (hrg : pR g ∈ s₀.rd ++ s₀.wr) (hdf : (pR h).Disjoint (pR f)) (hdg : (pR h).Disjoint (pR g)) + {F G : Poly} (hF : ∀ k < 256, (coeffAt s₀.mem f k).toNat = (F[k]!).val) + (hG : ∀ k < 256, (coeffAt s₀.mem g k).toNat = (G[k]!).val) + {core : List Instr} {Fv : BitVec 128 → BitVec 128 → BitVec 128 → BitVec 128} + (hcore : ∀ s : State, VConsts s → s.xmm .xmm11 = r2V → WP isa (.block core) s fun s' => + s'.xmm .xmm3 = Fv (s.xmm .xmm3) (s.xmm .xmm13) (s.xmm .xmm5) ∧ XOnly [.xmm12, .xmm3, .xmm2, .xmm4] s s') + (hY : laneSseBlock (toY core) = some core) + {R : Poly} {H : Nat → BitVec 32} (hH : ∀ k < 248, coeffAt s₀.mem h k = H k) + (hlane : ∀ i < 64, ∀ x y z : BitVec 128, (∀ e < 4, (dword x e).toNat = (F[4 * i + e]!).val) → + (∀ e < 4, (dword y e).toNat = (G[4 * i + e]!).val) → (∀ e < 4, dword z e = H (4 * i + e)) → + ∀ e < 4, (dword (Fv x y z) e).toNat = (R[4 * i + e]!).val) +include hwh hrf hrg hdf hdg hF hG hcore hY hH hlane + +theorem step {i : Nat} (hi : i < 31) {s : State} (hI : Inv h f g s₀ R i s) : + WP isa (.block (ymulLoads ++ toY core ++ ymulTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) s fun s' => + Inv h f g s₀ R (i + 1) s' ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by + have j0 : 8 * i + 8 ≤ 256 := by omega + have e1 : s.gpr .rsi + BitVec.ofNat 64 0 = coeffAddr f (8 * i) := by + rw [add_ofNat_zero, hI.rsi]; congr 2; omega + have e2 : s.gpr .rdx + BitVec.ofNat 64 0 = coeffAddr g (8 * i) := by + rw [add_ofNat_zero, hI.rdx]; congr 2; omega + have e3 : s.gpr .rdi + BitVec.ofNat 64 0 = coeffAddr h (8 * i) := by + rw [add_ofNat_zero, hI.rdi]; congr 2; omega + have mf : ∀ k < 256, coeffAt s.mem f k = coeffAt s₀.mem f k := fun k hk => + coeffAt_frame hI.frame (by simpa using hdf.symm) (by rw [n_eq]; exact hk) + have mg : ∀ k < 256, coeffAt s.mem g k = coeffAt s₀.mem g k := fun k hk => + coeffAt_frame hI.frame (by simpa using hdg.symm) (by rw [n_eq]; exact hk) + rw [show ymulLoads ++ toY core ++ ymulTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr) = + [.vmovdquLoad .l256 .xmm3 (at_ .rsi 0)] ++ [.vmovdquLoad .l256 .xmm13 (at_ .rdx 0)] ++ + [.vmovdquLoad .l256 .xmm5 (at_ .rdi 0)] ++ (toY core ++ (ymulTail ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) + by simp [ymulLoads, List.append_assoc], + WP.block_append_iff, WP.block_append_iff, WP.block_append_iff] + refine WP.mono (yld_ok (by rw [e1, hI.rd, hI.wr]; exact ⟨_, hrf, pR_contains32 f j0⟩)) fun s1 ⟨L1, o1⟩ => ?_ + refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr, e2, hI.rd, hI.wr]; exact ⟨_, hrg, pR_contains32 g j0⟩)) + fun s2 ⟨L2, o2⟩ => ?_ + have o12 := o1.trans o2 + refine WP.mono (yld_ok (by + rw [o12.rd, o12.wr, o12.gpr, e3, hI.rd, hI.wr]; exact f_in32 (List.mem_append_right _ hwh) j0)) + fun s3 ⟨L3, o3⟩ => ?_ + have o13 := o12.trans o3 + rw [WP.block_append_iff] + refine WP.mono (ylanes hY (P := fun l t => + t.xmm .xmm3 = Fv ((s3.proj l).xmm .xmm3) ((s3.proj l).xmm .xmm13) ((s3.proj l).xmm .xmm5)) + fun l hl => hcore _ (yonly_yconsts o13 hI.c (by decide) (by decide) l hl) + (by rw [State.proj_xmm, o13.lane _ (by decide) l hl]; exact hI.r2 l hl)) + fun s4 ⟨B4, o4⟩ => ?_ + have o14 := o13.trans o4 + have g4 : s4.gpr .rdi = coeffAddr h (8 * i) := by rw [o14.gpr, ← e3, add_ofNat_zero] + have w0 : InRegions s4.wr (s4.gpr .rdi) 32 := by rw [o14.wr, g4, hI.wr]; exact f_in32 hwh j0 + -- the lanes of the result + have hl : ∀ l < 2, ∀ e < 4, (dword (s4.lane .xmm3 l) e).toNat = (R[8 * i + 4 * l + e]!).val := by + intro l hl e he + rw [← State.proj_xmm, B4 l hl, State.proj_xmm, State.proj_xmm, State.proj_xmm, + o3.lane _ (by decide) l hl, o2.lane _ (by decide) l hl, o3.lane _ (by decide) l hl, L1 l hl, + L2 l hl, L3 l hl, o1.gpr, o1.mem, o2.gpr, o2.mem, o1.gpr, o1.mem, e1, e2, e3, + show 8 * i + 4 * l + e = 4 * (2 * i + l) + e by omega] + refine hlane (2 * i + l) (by omega) _ _ _ (fun e he => ?_) (fun e he => ?_) (fun e he => ?_) e he + · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, mf _ (by omega), hF _ (by omega), + show 8 * i + 4 * l + e = 4 * (2 * i + l) + e by omega] + · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, mg _ (by omega), hG _ (by omega), + show 8 * i + 4 * l + e = 4 * (2 * i + l) + e by omega] + · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, hI.rest _ (by omega) (by omega), + hH _ (by omega), show 8 * i + 4 * l + e = 4 * (2 * i + l) + e by omega] + simp only [ymulTail] + vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, + State.setMem_ymm, w0, sx32] + refine ⟨⟨?_, ?_, ?_, ?_, ?_, fun l hl => ⟨?_, ?_⟩, fun l hl => ?_, fun l hl => ?_, ?_, fun k hk => ?_, + fun k hk hk' => ?_⟩, ?_⟩ + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr] + rw [o14.gpr, hI.rdi]; exact ptr_step _ i 32 + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr] + rw [o14.gpr, hI.rsi]; exact ptr_step _ i 32 + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr] + rw [o14.gpr, hI.rdx]; exact ptr_step _ i 32 + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; rw [o14.rd, hI.rd] + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; rw [o14.wr, hI.wr] + · rw [State.proj_xmm]; simp only [lane_setReg, lane_setFlags, State.setMem_lane] + exact (yonly_yconsts o14 hI.c (by decide) (by decide) l hl).q + · rw [State.proj_xmm]; simp only [lane_setReg, lane_setFlags, State.setMem_lane] + exact (yonly_yconsts o14 hI.c (by decide) (by decide) l hl).qinv + · simp only [lane_setReg, lane_setFlags, State.setMem_lane]; rw [o14.lane _ (by decide) l hl]; exact hI.r2 l hl + · simp only [lane_setReg, lane_setFlags, State.setMem_lane]; rw [o14.lane _ (by decide) l hl]; exact hI.x6 l hl + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem] + rw [g4, o14.mem]; exact hI.frame.writeW (List.mem_singleton_self _) _ (pR_contains32 _ j0) + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem] + rw [g4, o14.mem, coeffAt_write256 _ _ j0 _ (by omega)] + split + · rename_i hk' + rw [State.ymm, extract_ymm _ _ (by omega)] + split + · rename_i h4 + have := hl 0 (by decide) (k - 8 * i) h4 + rw [show 8 * i + 4 * 0 + (k - 8 * i) = k by omega] at this + exact this + · rename_i h4 + have := hl 1 (by decide) (k - 8 * i - 4) (by omega) + rw [show 8 * i + 4 * 1 + (k - 8 * i - 4) = k by omega] at this + exact this + · exact hI.done k (by omega) + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem] + rw [g4, o14.mem, coeffAt_write256 _ _ j0 _ hk, ifn (by omega)] + exact hI.rest k hk (by omega) + · exact ⟨by rw [o14.gpr], by rw [o14.gpr]⟩ + +theorem loop_ok (hdi : s₀.gpr .rdi = h) (hsi : s₀.gpr .rsi = f) (hdx : s₀.gpr .rdx = g) (hc : YConsts s₀) + (h11 : ∀ l < 2, s₀.lane .xmm11 l = r2V) : + WP isa (rcxLoop 31 (ymulLoads ++ toY core ++ ymulTail)) s₀ (Inv h f g s₀ R 31) := + wp_rcxLoopY (N := 31) (by decide) (by decide) _ (fun u o hy _ => + have lu : ∀ r l, u.lane r l = s₀.lane r l := fun r l => by simp only [State.lane]; rw [o.xmm, hy] + ⟨by rw [o.keep.gpr (by decide), hdi, Nat.mul_zero, add_ofNat_zero], + by rw [o.keep.gpr (by decide), hsi, Nat.mul_zero, add_ofNat_zero], + by rw [o.keep.gpr (by decide), hdx, Nat.mul_zero, add_ofNat_zero], o.keep.2.1, o.keep.2.2, + fun l hl => ⟨by rw [State.proj_xmm, lu]; exact (hc l hl).q, by rw [State.proj_xmm, lu]; exact (hc l hl).qinv⟩, + fun l hl => by rw [lu]; exact h11 l hl, fun l _ => lu _ l, + by rw [o.mem]; exact Frame.refl _ _, fun k hk => absurd hk (by omega), fun k _ _ => by rw [o.mem]⟩) + fun i hi u hI => step hwh hrf hrg hdf hdg hF hG hcore hY hH hlane hi hI + +omit hwh hH in +/-- The last eight coefficients, with those of `h` in `ymm6`. -/ +theorem last {s : State} (hI : Inv h f g s₀ R 31 s) + (hz : ∀ l < 2, ∀ e < 4, dword (s₀.lane .xmm6 l) e = H (248 + 4 * l + e)) : + WP isa (.block (ymulLast core)) s fun s' => + (∀ l < 2, ∀ e < 4, (dword (s'.lane .xmm3 l) e).toNat = (R[248 + 4 * l + e]!).val) ∧ s'.gpr = s.gpr ∧ + s'.mem = s.mem ∧ s'.rd = s.rd ∧ s'.wr = s.wr := by + have j0 : 8 * 31 + 8 ≤ 256 := by decide + have e1 : s.gpr .rsi + BitVec.ofNat 64 0 = coeffAddr f 248 := by rw [add_ofNat_zero, hI.rsi] + have e2 : s.gpr .rdx + BitVec.ofNat 64 0 = coeffAddr g 248 := by rw [add_ofNat_zero, hI.rdx] + have mf : ∀ k < 256, coeffAt s.mem f k = coeffAt s₀.mem f k := fun k hk => + coeffAt_frame hI.frame (by simpa using hdf.symm) (by rw [n_eq]; exact hk) + have mg : ∀ k < 256, coeffAt s.mem g k = coeffAt s₀.mem g k := fun k hk => + coeffAt_frame hI.frame (by simpa using hdg.symm) (by rw [n_eq]; exact hk) + have hY' : laneSseBlock ([.vop (.vmovdqa .l256 .xmm5 .xmm6)] ++ toY core) = some (xmov .xmm5 .xmm6 :: core) := by + show (match laneSse _, laneSseBlock (toY core) with | some a, some b => some (a ++ b) | _, _ => none) = _ + rw [hY]; rfl + rw [ymulLast, show ∀ a b c : Instr, ∀ l : List Instr, [a, b, c] ++ l = [a] ++ [b] ++ ([c] ++ l) from + fun _ _ _ _ => rfl, WP.block_append_iff, WP.block_append_iff] + refine WP.mono (yld_ok (by rw [e1, hI.rd, hI.wr]; exact ⟨_, hrf, pR_contains32 f j0⟩)) fun s1 ⟨L1, o1⟩ => ?_ + refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr, e2, hI.rd, hI.wr]; exact ⟨_, hrg, pR_contains32 g j0⟩)) + fun s2 ⟨L2, o2⟩ => ?_ + have o12 := o1.trans o2 + refine WP.mono (ylanes hY' (rs := [.xmm5, .xmm12, .xmm3, .xmm2, .xmm4]) (P := fun l t => + t.xmm .xmm3 = Fv ((s2.proj l).xmm .xmm3) ((s2.proj l).xmm .xmm13) ((s2.proj l).xmm .xmm6)) + fun l hl => ?_) fun s3 ⟨B3, o3⟩ => ⟨fun l hl e he => ?_, (o12.trans o3).gpr, (o12.trans o3).mem, + (o12.trans o3).rd, (o12.trans o3).wr⟩ + · have c2 := yonly_yconsts o12 hI.c (by decide) (by decide) l hl + rw [show xmov .xmm5 .xmm6 :: core = [xmov .xmm5 .xmm6] ++ core from rfl, WP.block_append_iff] + vrund [eval_movdqa] + refine WP.mono (hcore _ (c2.setXmm (by decide) (by decide) _) + (by rw [xmm_setXmm, ifn (by decide), State.proj_xmm, o12.lane _ (by decide) l hl]; exact hI.r2 l hl)) + fun t ⟨ht, ot⟩ => ⟨?_, ?_⟩ + · rw [ht, xmm_setXmm, xmm_setXmm, xmm_setXmm, ifn (by decide), ifn (by decide), ifp rfl] + · refine (XOnly.trans (⟨⟨rfl, rfl, rfl, rfl, rfl⟩, fun r hr => ?_⟩ : + XOnly [.xmm5] (s2.proj l) ((s2.proj l).setXmm .xmm5 ((s2.proj l).xmm .xmm6))) ot).mono + (rs' := [.xmm5, .xmm12, .xmm3, .xmm2, .xmm4]) (by simp) + rw [xmm_setXmm, ifn (by simpa using hr)] + · rw [← State.proj_xmm, B3 l hl, State.proj_xmm, State.proj_xmm, State.proj_xmm, + o2.lane _ (by decide) l hl, L1 l hl, L2 l hl, o1.gpr, o1.mem, e1, e2, + o12.lane _ (by decide) l hl, hI.x6 l hl, show 248 + 4 * l + e = 4 * (62 + l) + e by omega] + refine hlane (62 + l) (by omega) _ _ _ (fun e he => ?_) (fun e he => ?_) (fun e he => ?_) e he + · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, mf _ (by omega), hF _ (by omega), + show 248 + 4 * l + e = 4 * (62 + l) + e by omega] + · rw [dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, mg _ (by omega), hG _ (by omega), + show 248 + 4 * l + e = 4 * (62 + l) + e by omega] + · rw [hz l hl e he, show 248 + 4 * l + e = 4 * (62 + l) + e by omega] + +end + + +/-- The whole function, from its precondition, with a `core` whose lanes are +those of `t`. -/ +theorem fn_ok {t : Poly → Poly → Poly → Poly} {hPre : Mem → Addr → Prop} {σ : State} (hp : (mulK t hPre).pre σ) + {core : List Instr} {Fv : BitVec 128 → BitVec 128 → BitVec 128 → BitVec 128} + (hcore : ∀ s : State, VConsts s → s.xmm .xmm11 = r2V → WP isa (.block core) s fun s' => + s'.xmm .xmm3 = Fv (s.xmm .xmm3) (s.xmm .xmm13) (s.xmm .xmm5) ∧ XOnly [.xmm12, .xmm3, .xmm2, .xmm4] s s') + (hY : laneSseBlock (toY core) = some core) + (hlane : ∀ i < 64, ∀ x y z : BitVec 128, + (∀ e < 4, (dword x e).toNat = ((polyAt σ.mem (σ.gpr .rsi))[4 * i + e]!).val) → + (∀ e < 4, (dword y e).toNat = ((polyAt σ.mem (σ.gpr .rdx))[4 * i + e]!).val) → + (∀ e < 4, dword z e = coeffAt σ.mem (σ.gpr .rdi) (4 * i + e)) → + ∀ e < 4, (dword (Fv x y z) e).toNat = ((t (polyAt σ.mem (σ.gpr .rdi)) (polyAt σ.mem (σ.gpr .rsi)) + (polyAt σ.mem (σ.gpr .rdx)))[4 * i + e]!).val) + (hk : writesOnly [.rax, .rdi, .rsi, .rdx, .rcx] + (.seq (.block ymulPro) (.seq (rcxLoop 31 (ymulLoads ++ toY core ++ ymulTail)) (.block (ymulLast core)))) = + true) : + WP isa (ymulFn core) σ fun s' => Keep [.r8, .rax, .r11, .rax, .rdi, .rsi, .rdx, .rcx] σ s' ∧ + Frame [pR (σ.gpr .rdi)] σ.mem s'.mem ∧ (mulK t hPre).post σ s' := by + obtain ⟨hrd, hwr, hdf, hdg, -, -, -, -, redf, redg⟩ := hp + generalize eh : σ.gpr .rdi = h at * + generalize ef : σ.gpr .rsi = f at * + generalize eg : σ.gpr .rdx = g at * + let R := t (polyAt σ.mem h) (polyAt σ.mem f) (polyAt σ.mem g) + have hwh : pR h ∈ σ.wr := by rw [hwr]; exact List.mem_singleton_self _ + simp only [ymulFn] + rw [show ∀ a b : Instr, [a, b] = [a] ++ [b] from fun _ _ => rfl] + refine WP.seq (WP.mono (Q := fun (s1 : State) => s1.gpr .r8 = h ∧ + (∀ l < 2, s1.lane .xmm6 l = σ.mem.readW (coeffAddr h 248 + BitVec.ofNat 64 (16 * l)) 128) ∧ + Keep [.r8] σ s1 ∧ s1.mem = σ.mem) ?_ fun s1 ⟨h8, h6, k1, m1⟩ => ?_) + · rw [WP.block_append_iff] + refine WP.mono (Q := fun (s0 : State) => s0.gpr .r8 = h ∧ s0.gpr .rdi = h ∧ GOnly [.r8] σ s0 ∧ + s0.ymmHi = σ.ymmHi) (by vrund [eh, RegUpd.ymmHi_setReg]; gonlyd) + fun s0 ⟨h80, hd0, o0, y0⟩ => ?_ + refine WP.mono (yld_ok (by + rw [hd0, o0.keep.2.1, o0.keep.2.2] + exact ⟨_, List.mem_append_right _ hwh, Offset.contains_base h (by omega) (by omega)⟩)) + fun s1 ⟨L1, o1⟩ => ⟨by rw [o1.gpr, h80], fun l hl => by rw [L1 l hl, hd0, o0.mem], + ⟨fun r hr => by rw [o1.gpr, o0.keep.gpr hr], by rw [o1.rd, o0.keep.2.1], by rw [o1.wr, o0.keep.2.2]⟩, + by rw [o1.mem, o0.mem]⟩ + have hw1 : pR h ∈ s1.wr := by rw [k1.2.2]; exact hwh + refine WP.seq (WP.mono (withMxcsrH_ok (r := .r8) ⟨by decide, by decide⟩ [.rax, .rdi, .rsi, .rdx, .rcx] + ⟨by decide, by decide⟩ h8 hw1 hk (Q := fun (s3 : State) => Keep [.rax, .r11, .rax, .rdi, .rsi, .rdx, .rcx] s1 s3 ∧ + s3.gpr .rdi = coeffAddr h 248 ∧ Frame [pR h] σ.mem s3.mem ∧ + (∀ k < 248, (coeffAt s3.mem h k).toNat = (R[k]!).val) ∧ + ∀ l < 2, ∀ e < 4, (dword (s3.lane .xmm3 l) e).toNat = (R[248 + 4 * l + e]!).val) + fun s2 k2 f2 x2 y2 => ?_) fun s4 ⟨s3, ⟨kk, hdi, fr, dn, ln⟩, f4, k4, x4, y4⟩ => ?_) + · have fσ2 : Frame [mxH h] σ.mem s2.mem := by rw [← m1]; exact f2 + have l2 : ∀ r l, s2.lane r l = s1.lane r l := fun r l => by simp only [State.lane]; rw [x2, y2] + refine WP.mono (WP.keep [.rax, .rdi, .rsi, .rdx, .rcx] (WP.seq (WP.mono (ymulPro_ok s2) + fun w ⟨cw, xw, x6, kw, mw⟩ => ?_) (Q := fun (s3 : State) => s3.gpr .rdi = coeffAddr h 248 ∧ + Frame [pR h] σ.mem s3.mem ∧ (∀ k < 248, (coeffAt s3.mem h k).toNat = (R[k]!).val) ∧ + ∀ l < 2, ∀ e < 4, (dword (s3.lane .xmm3 l) e).toNat = (R[248 + 4 * l + e]!).val)) hk) + fun s3 ⟨q3, kk⟩ => ⟨k2.trans kk, q3⟩ + have gw : ∀ r, r ≠ .rax → r ≠ .r11 → r ≠ .r8 → w.gpr r = σ.gpr r := fun r h1 h2 h3 => by + rw [kw.gpr (by simpa using h1), k2.gpr (by simp [h1, h2]), k1.gpr (by simpa using h3)] + have rw' : w.rd = σ.rd ∧ w.wr = σ.wr := + ⟨kw.2.1.trans (k2.2.1.trans k1.2.1), kw.2.2.trans (k2.2.2.trans k1.2.2)⟩ + have fσw : Frame [mxH h] σ.mem w.mem := by rw [mw]; exact fσ2 + have fσw' : Frame [pR h] σ.mem w.mem := + Frame.sub fσw fun r hr => ⟨pR h, List.mem_singleton_self _, by + simp only [List.mem_singleton] at hr; subst hr; exact mxH_sub h⟩ + refine WP.seq (WP.mono (loop_ok (s₀ := w) (h := h) (f := f) (g := g) (R := R) (F := polyAt σ.mem f) + (G := polyAt σ.mem g) (H := coeffAt σ.mem h) + (by rw [rw'.2]; exact hwh) (by rw [rw'.1, hrd]; simp) (by rw [rw'.1, hrd]; simp) hdf hdg + (fun k hk => by + rw [coeffAt_frame fσw' (by simpa using hdf.symm) (by rw [n_eq]; exact hk), + polyAt_val redf (by rw [n_eq]; exact hk)]) + (fun k hk => by + rw [coeffAt_frame fσw' (by simpa using hdg.symm) (by rw [n_eq]; exact hk), + polyAt_val redg (by rw [n_eq]; exact hk)]) + hcore hY (fun k hk => Mul.coeffAt_mxH fσw (by omega)) hlane + (by rw [gw _ (by decide) (by decide) (by decide), eh]) (by rw [gw _ (by decide) (by decide) (by decide), ef]) + (by rw [gw _ (by decide) (by decide) (by decide), eg]) cw xw) fun s hI => ?_) + refine WP.mono (last (s₀ := w) (h := h) (f := f) (g := g) (R := R) (F := polyAt σ.mem f) + (G := polyAt σ.mem g) (H := coeffAt σ.mem h) + (by rw [rw'.1, hrd]; simp) (by rw [rw'.1, hrd]; simp) hdf hdg + (fun k hk => by + rw [coeffAt_frame fσw' (by simpa using hdf.symm) (by rw [n_eq]; exact hk), + polyAt_val redf (by rw [n_eq]; exact hk)]) + (fun k hk => by + rw [coeffAt_frame fσw' (by simpa using hdg.symm) (by rw [n_eq]; exact hk), + polyAt_val redg (by rw [n_eq]; exact hk)]) + hcore hY hlane hI (fun l hl e he => by + rw [x6 l hl, l2, h6 l hl, dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq])) + fun s3 ⟨ln, g3, m3, _, _⟩ => ?_ + refine ⟨by rw [g3, hI.rdi], ?_, fun k hk => by rw [m3]; exact hI.done k (by omega), ln⟩ + rw [m3] + exact Frame.trans fσw' hI.frame + · have k14 := (k1.trans kk).trans k4 + have hdi4 : s4.gpr .rdi = coeffAddr h 248 := by rw [k4.gpr (by simp), hdi] + have wh4 : InRegions s4.wr (s4.gpr .rdi) 32 := by + rw [hdi4, k14.2.2]; exact f_in32 hwh (by omega) + have l4 : ∀ r l, s4.lane r l = s3.lane r l := fun r l => by simp only [State.lane]; rw [x4, y4] + simp only [yepi, List.singleton_append] + vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, + State.setMem_ymm, wh4] + have f4' : Frame [pR h] s3.mem s4.mem := Frame.sub f4 fun r hr => ⟨pR h, List.mem_singleton_self _, by + simp only [List.mem_singleton] at hr; subst hr; exact mxH_sub h⟩ + refine ⟨k14.mono (by simp), (fr.trans f4').writeW (List.mem_singleton_self _) _ + (by rw [hdi4]; exact pR_contains32 h (by omega)), ?_⟩ + dsimp only [mulK] + rw [eh, ef, eg, show ∀ s : State, (VOp.vzeroupper.exec s).mem = s.mem from fun _ => rfl, State.setMem_mem] + refine polyIs_of_toNat fun k hk => ?_ + rw [n_eq] at hk + rw [hdi4, coeffAt_write256 _ _ (j := 248) (by decide) _ hk] + split + · rw [State.ymm, extract_ymm _ _ (by omega)] + split + · rename_i h1 h4 + have := ln 0 (by decide) (k - 248) h4 + rw [show 248 + 4 * 0 + (k - 248) = k by omega, ← l4] at this + exact this + · rename_i h1 h4 + have := ln 1 (by decide) (k - 248 - 4) (by omega) + rw [show 248 + 4 * 1 + (k - 248 - 4) = k by omega, ← l4] at this + exact this + · rw [Mul.coeffAt_mxH f4 (by omega)] + exact dn k (by omega) + +end YMul + +/-! ## The functions -/ + +theorem mulY_correct (s : State) (hs : mulK'.pre s) : + ∃ t s', Exec isa mulAvx2 s t s' ∧ abiPreserved s s' ∧ mulK'.post s s' := by + obtain ⟨t, s', he, hk, hf, hq⟩ := YMul.fn_ok hs (core := mulCore) (Fv := fun x y _ => mulV x y) + (fun s hc h11 => mulCore_ok hc h11) lane_mulCore + (fun i hi x y z hx hy _ e he => by + rw [mul_get _ _ (by rw [n_eq]; omega)] + exact mul_lane hx hy he) + (by decide +kernel) + exact ⟨t, s', he, abiPreserved_of_ctl (by decide +kernel) he (gprPreserved_of hk (by decide) hf + (by simpa using hs.2.2.2.2.1)), hq⟩ + +theorem mulAddY_correct (s : State) (hs : mulAddK.pre s) : + ∃ t s', Exec isa mulAddAvx2 s t s' ∧ abiPreserved s s' ∧ mulAddK.post s s' := by + obtain ⟨t, s', he, hk, hf, hq⟩ := YMul.fn_ok hs (core := mulAddCore) (Fv := mulAddV) + (fun s hc h11 => mulAddCore_ok hc h11) lane_mulAddCore + (fun i hi x y z hx hy hz e he => by + rw [add_get _ _ (by rw [n_eq]; omega), mul_get _ _ (by rw [n_eq]; omega)] + exact mulAdd_lane hx hy (c := fun e => (polyAt s.mem (s.gpr .rdi))[4 * i + e]!) + (fun e he => by rw [hz e he, polyAt_val hs.2.2.2.2.2.2.2.1 (by rw [n_eq]; omega)]) he) + (by decide +kernel) + exact ⟨t, s', he, abiPreserved_of_ctl (by decide +kernel) he (gprPreserved_of hk (by decide) hf + (by simpa using hs.2.2.2.2.1)), hq⟩ + +theorem mulY_ct : ConstantTime isa mulK'.pre mulK'.pub mulAvx2 := + VG.Taint.constantTime (A := taint) mulτ mul_agree (by taint_decide) + +theorem mulAddY_ct : ConstantTime isa mulAddK.pre mulAddK.pub mulAddAvx2 := + VG.Taint.constantTime (A := taint) mulτ mul_agree (by taint_decide) + +theorem mulY_verified : Verified X86_64.target mulAvx2 (Spec.MlDsa.mulContract X86_64.abi) := + Verified.of_correct mulY_correct mulY_ct (by + mldsa_implies [Spec.MlDsa.mulContract, Spec.MlDsa.mulSig, mulK, X86_64.abi, X86_64.argRegs] + [mulSat] using mulSat) + +theorem mulAddY_verified : Verified X86_64.target mulAddAvx2 (Spec.MlDsa.mulAddContract X86_64.abi) := + Verified.of_correct mulAddY_correct mulAddY_ct (by + mldsa_implies [Spec.MlDsa.mulAddContract, Spec.MlDsa.mulSig, mulK, X86_64.abi, X86_64.argRegs] + [mulSat] using mulSat) + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YNtt.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YNtt.lean new file mode 100644 index 000000000..4e06aaf10 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YNtt.lean @@ -0,0 +1,419 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YLay21 +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv + +/-! +# ML-DSA on x86-64: `vg_mldsa_ntt_avx2` and `vg_mldsa_inv_ntt_avx2` + +Untrusted: everything here is checked by Lean. As `vg_mldsa_ntt` and +`vg_mldsa_inv_ntt` (`Ntt.lean`, `NttInv.lean`): ML-KEM's `withMxcsr` runs +the code from any MXCSR and keeps what it does (`ymx_correct`); the +prologue leaves the table of zetas in `scratch` and the constants in both +lanes (`ypro_ok`); each layer is `nttLayer` or `nttInvLayer` (`ylay_ok`, +`ylay4_ok`, `ylay2_ok`, `ylay1_ok`), `NTT⁻¹` then multiplies every +coefficient by `8347681 = 256⁻¹ mod q` (`yscale_ok`), and `vzeroupper` +keeps the memory (`LIY.epi`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes yld_ok yconst_ok ifp ifn sel GOnly WP.keep writesOnly + gprPreserved_of withMxcsr_ok mxR add_ofNat_zero wp_rcxLoopY lane_setReg lane_setFlags State.setMem_ymm sx32) +open VG.Impl.MlKem.X86_64 (xb xmov toY rcxLoop yconst) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt PolyIs zetas ntt nttInv) + +theorem lane_vbfly : laneSseBlock (toY vbfly) = some vbfly := by decide +kernel +theorem lane_vibfly : laneSseBlock (toY vibfly) = some vibfly := by decide +kernel +theorem lane_core2f : laneSseBlock (toY (gath2 ++ vbfly ++ scat2)) = some (gath2 ++ vbfly ++ scat2) := by + decide +kernel +theorem lane_core2i : laneSseBlock (toY (gath2 ++ vibfly ++ scat2)) = some (gath2 ++ vibfly ++ scat2) := by + decide +kernel +theorem lane_core1f : laneSseBlock (toY (gath1 ++ vbfly ++ scat1)) = some (gath1 ++ vbfly ++ scat1) := by + decide +kernel +theorem lane_core1i : laneSseBlock (toY (gath1 ++ vibfly ++ scat1)) = some (gath1 ++ vibfly ++ scat1) := by + decide +kernel +theorem lane_vmul3 : laneSseBlock (toY (vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ vcsub .xmm3 .xmm2)) = + some (vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ vcsub .xmm3 .xmm2) := by decide +kernel + +/-! ## The prologue and the layers -/ + +/-- The table of zetas and the constants. -/ +theorem ypro_ok {sP : Addr} {s : State} (hsi : s.gpr .rsi = sP) (hw : pR sP ∈ s.wr) : + WP isa (.block ypro) s fun s' => Tab zmTab s'.mem sP 256 ∧ YConsts s' ∧ + Frame [pR sP] s.mem s'.mem ∧ Keep [.r9, .rax] s s' ∧ s'.mxcsr = s.mxcsr := by + rw [ypro, WP.block_append_iff] + refine WP.mono (dwordTab_ok zmTab (fun k => Nat.lt_trans (zmTab_lt k) (by decide)) (by decide) hsi hw) + fun s1 ⟨hT, hf, k1, x1, _⟩ => WP.mono (yconsts_ok s1) fun s2 ⟨hc, k2, m2, x2, _⟩ => + ⟨by rw [m2]; exact hT, hc, by rw [m2]; exact hf, (k1.trans k2).mono (by simp), by rw [x2, x1]⟩ + +/-- Between the layers: the polynomial `F` at `fP`, the table at `sP`, and +the constants in both lanes. -/ +structure LIY (fP sP : Addr) (s₀ : State) (F : Poly) (s : State) : Prop where + P : PolyIs s.mem fP F + T : Tab zmTab s.mem sP 256 + c : YConsts s + keep : Keep [.rax, .rcx, .rdx, .r8] s₀ s + frame : Frame [pR fP] s₀.mem s.mem + +/-- A layer, then `c`. -/ +theorem LIY.seq {fP sP : Addr} {s₀ : State} (hdi : s₀.gpr .rdi = fP) (hsi : s₀.gpr .rsi = sP) + (hwf : pR fP ∈ s₀.wr) (hw : pR sP ∈ s₀.wr) (hd : (pR sP).Disjoint (pR fP)) {l c : Prog isa} + {F F' : Poly} {Q : State → Prop} + (hl : ∀ s, YConsts s → s.gpr .rdi = fP → s.gpr .rsi = sP → PolyIs s.mem fP F → Tab zmTab s.mem sP 256 → + pR fP ∈ s.wr → pR sP ∈ s.wr → WP isa l s fun s' => PolyIs s'.mem fP F' ∧ BInvY fP s s') + (hc : ∀ s, LIY fP sP s₀ F' s → WP isa c s Q) {s : State} (hI : LIY fP sP s₀ F s) : + WP isa (.seq l c) s Q := + WP.seq (WP.mono (hl s hI.c (by rw [hI.keep.gpr (by decide), hdi]) (by rw [hI.keep.gpr (by decide), hsi]) hI.P + hI.T (by rw [hI.keep.2.2]; exact hwf) (by rw [hI.keep.2.2]; exact hw)) + fun s' ⟨hS, hb⟩ => hc s' ⟨hS, hI.T.frame hb.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) + (by decide), hb.consts, (hI.keep.trans hb.keep).mono (by decide), hI.frame.trans hb.frame⟩) + +/-- `vzeroupper` keeps the memory. -/ +theorem LIY.epi {fP sP : Addr} {s₀ : State} {F : Poly} {s : State} (hI : LIY fP sP s₀ F s) : + WP isa (.block yepi) s fun s' => PolyIs s'.mem fP F ∧ Frame [pR fP] s₀.mem s'.mem := + WP.mono (Q := fun (u : State) => u.mem = s.mem) (by simp only [yepi]; vrund; rfl) + fun u hm => by rw [hm]; exact ⟨hI.P, hI.frame⟩ + +/-- A layer of `NTT` with `len ≥ 8`, whose first zeta is `zetas k`. -/ +theorem yfwdLay_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) (len k : Nat) + (hlen : len ∈ [8, 16, 32, 64, 128]) (hk : 128 / len = k) {F : Poly} (s : State) (hc : YConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (ylay vbfly len k 4) s fun s' => PolyIs s'.mem fP (nttLayer F len) ∧ BInvY fP s s' := by + rw [nttLayer_eq] + exact ylay_ok vbfly_spec lane_vbfly nttBlk_ok hlen 4 (fun c => 128 / len + c) (by rw [hk]; rfl) + (fun c hc => by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hlen + rcases hlen with rfl | rfl | rfl | rfl | rfl <;> omega) + (fun c _ => step_fwd _ _ 1 (by decide)) hc hdi hsi hS hT hwf hw hd + +theorem yfwdLay4_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (ylay4 vbfly 32 0x00 0x55 8) s fun s' => PolyIs s'.mem fP (nttLayer F 4) ∧ BInvY fP s s' := by + have h0 : ∀ e < 4, sel 0x00 e = 0 := by decide + have h5 : ∀ e < 4, sel 0x55 e = 1 := by decide + rw [nttLayer_eq, show 128 / 4 = 32 from rfl] + exact ylay4_ok vbfly_spec lane_vbfly nttBlk_ok 32 0x00 0x55 8 (fun c => 32 + c) (fun m => 32 + 2 * m) rfl + (fun m _ => by omega) (fun m _ e he => ⟨by rw [h0 e he]; omega, by rw [h5 e he]; omega⟩) + (fun m _ => (step_fwd _ _ 2 (by decide)).trans (congrArg _ (by omega))) hc hdi hsi hS hT hwf hw hd + +theorem yfwdLay2_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (ylay2 vbfly 64 0xA0 0xF5 16) s fun s' => PolyIs s'.mem fP (nttLayer F 2) ∧ BInvY fP s s' := by + have hA : ∀ e < 4, sel 0xA0 e = 2 * (e / 2) := by decide + have hF : ∀ e < 4, sel 0xF5 e = 1 + 2 * (e / 2) := by decide + rw [nttLayer_eq, show 128 / 2 = 64 from rfl] + exact ylay2_ok nttBlk_ok vbfly_spec lane_core2f 64 0xA0 0xF5 16 (fun c => 64 + c) (fun i => 64 + 4 * i) rfl + (fun i _ => by omega) (fun i _ e he => ⟨by rw [hA e he]; omega, by rw [hF e he]; omega⟩) + (fun i _ => (step_fwd _ _ 4 (by decide)).trans (congrArg _ (by omega))) hc hdi hsi hS hT hwf hw hd + +theorem yfwdLay1_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (ylay1 vbfly 128 yzeta8 32) s fun s' => PolyIs s'.mem fP (nttLayer F 1) ∧ BInvY fP s s' := by + rw [nttLayer_eq, show 128 / 1 = 128 from rfl] + exact ylay1_ok nttBlk_ok vbfly_spec lane_core1f 128 yzeta8 32 (fun c => 128 + c) (fun i => 128 + 8 * i) rfl + (fun i _ => by omega) + (fun i hi s h8 hin hT => WP.mono (yzeta8_ok (by omega) h8 hin hT) fun _ ⟨z, o⟩ => + ⟨fun l hl => ⟨(z l hl).1.congr fun e he => congrArg zetas (by omega), (z l hl).2⟩, o⟩) + (fun i _ => (step_fwd _ _ 8 (by decide)).trans (congrArg _ (by omega))) hc hdi hsi hS hT hwf hw hd + +theorem yinvLay_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) (len k : Nat) + (hlen : len ∈ [8, 16, 32, 64, 128]) (hk : 256 / len - 1 = k) {F : Poly} (s : State) (hc : YConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (ylay vibfly len k (-4)) s fun s' => PolyIs s'.mem fP (nttInvLayer F len) ∧ BInvY fP s s' := by + have hl : 128 / len ≥ 1 ∧ 256 / len = 2 * (128 / len) ∧ 256 / len ≤ 32 := by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hlen + rcases hlen with rfl | rfl | rfl | rfl | rfl <;> decide + rw [nttInvLayer_eq] + exact ylay_ok vibfly_spec lane_vibfly nttInvBlk_ok hlen (-4) (fun c => 256 / len - 1 - c) (by rw [hk]; rfl) + (fun c _ => by omega) + (fun c hc' => (congrArg (· + _) (congrArg (coeffAddr sP) (show 256 / len - 1 - c = + 256 / len - 1 - (c + 1) + 1 by omega))).trans (step_bwd _ _ 1 (by decide))) + hc hdi hsi hS hT hwf hw hd + +theorem yinvLay4_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (ylay4 vibfly 62 0x55 0x00 (-8)) s fun s' => PolyIs s'.mem fP (nttInvLayer F 4) ∧ BInvY fP s s' := by + have h0 : ∀ e < 4, sel 0x00 e = 0 := by decide + have h5 : ∀ e < 4, sel 0x55 e = 1 := by decide + rw [nttInvLayer_eq, show 256 / 4 - 1 = 63 from rfl, show 128 / 4 = 32 from rfl] + exact ylay4_ok vibfly_spec lane_vibfly nttInvBlk_ok 62 0x55 0x00 (-8) (fun c => 63 - c) (fun m => 62 - 2 * m) + rfl (fun m _ => by omega) (fun m _ e he => ⟨by rw [h5 e he]; omega, by rw [h0 e he]; omega⟩) + (fun m _ => (congrArg (· + _) (congrArg (coeffAddr sP) (show 62 - 2 * m = 62 - 2 * (m + 1) + 2 by + omega))).trans (step_bwd _ _ 2 (by decide))) hc hdi hsi hS hT hwf hw hd + +theorem yinvLay2_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (ylay2 vibfly 124 0x5F 0x0A (-16)) s fun s' => PolyIs s'.mem fP (nttInvLayer F 2) ∧ + BInvY fP s s' := by + have hA : ∀ e < 4, sel 0x5F e = 3 - 2 * (e / 2) := by decide + have hB : ∀ e < 4, sel 0x0A e = 2 - 2 * (e / 2) := by decide + rw [nttInvLayer_eq, show 256 / 2 - 1 = 127 from rfl, show 128 / 2 = 64 from rfl] + exact ylay2_ok nttInvBlk_ok vibfly_spec lane_core2i 124 0x5F 0x0A (-16) (fun c => 127 - c) + (fun i => 124 - 4 * i) rfl (fun i _ => by omega) + (fun i _ e he => ⟨by rw [hA e he]; omega, by rw [hB e he]; omega⟩) + (fun i _ => (congrArg (· + _) (congrArg (coeffAddr sP) (show 124 - 4 * i = 124 - 4 * (i + 1) + 4 by + omega))).trans (step_bwd _ _ 4 (by decide))) hc hdi hsi hS hT hwf hw hd + +theorem yinvLay1_ok {fP sP : Addr} (hd : (pR sP).Disjoint (pR fP)) {F : Poly} (s : State) (hc : YConsts s) + (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP F) (hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) : + WP isa (ylay1 vibfly 248 yzeta8R (-32)) s fun s' => PolyIs s'.mem fP (nttInvLayer F 1) ∧ + BInvY fP s s' := by + rw [nttInvLayer_eq, show 256 / 1 - 1 = 255 from rfl, show 128 / 1 = 128 from rfl] + exact ylay1_ok nttInvBlk_ok vibfly_spec lane_core1i 248 yzeta8R (-32) (fun c => 255 - c) + (fun i => 248 - 8 * i) rfl (fun i _ => by omega) + (fun i hi s h8 hin hT => WP.mono (yzeta8R_ok (by omega) h8 hin hT) fun _ ⟨z, o⟩ => + ⟨fun l hl => ⟨(z l hl).1.congr fun e he => congrArg zetas (by omega), (z l hl).2⟩, o⟩) + (fun i _ => (congrArg (· + _) (congrArg (coeffAddr sP) (show 248 - 8 * i = 248 - 8 * (i + 1) + 8 by + omega))).trans (step_bwd _ _ 8 (by decide))) hc hdi hsi hS hT hwf hw hd + +/-! ## The multiplication by `256⁻¹` -/ + +/-- The coefficients of `G` before `8i` multiplied by `8347681`. -/ +def YScaled (m : Mem) (fP : Addr) (G : Poly) (i : Nat) : Prop := + ∀ k < 256, (coeffAt m fP k).toNat = (if k < 8 * i then G[k]! * 8347681 else G[k]!).val + +/-- `8347681 · 2³² mod q` in each doubleword. -/ +theorem scale_lanes : ZLanes (ofDwords 16382#32 16382#32 16382#32 16382#32) (fun _ => 8347681) ∧ + ZOdd (ofDwords 16382#32 16382#32 16382#32 16382#32) (ofDwords 16382#32 16382#32 16382#32 16382#32) := + ⟨fun i hi => by rcases cases4 hi with rfl | rfl | rfl | rfl <;> decide, + fun j hj => by rcases (by omega : j = 0 ∨ j = 1) with rfl | rfl <;> decide⟩ + +theorem mov12_ok (t : State) : + WP isa (.block [xmov .xmm12 .xmm13]) t fun t' => t'.xmm .xmm12 = t.xmm .xmm13 ∧ XOnly [.xmm12] t t' := by + simp only [xmov, xb] + vrun [eval_movdqa] + exact ⟨by first | trivial | simp, by xonly⟩ + +/-- The body of the loop of `yscale`. -/ +abbrev sbodyY : List Instr := + [.vmovdquLoad .l256 .xmm3 (at_ .rdx 0)] ++ toY (vmont .xmm3 .xmm13 .xmm12 .xmm2 .xmm4 ++ vcsub .xmm3 .xmm2) ++ + [.vmovdquStore .l256 (at_ .rdx 0) .xmm3, .alu .add .rdx (.imm 32)] ++ [.alu .sub .rcx (.imm 1)] + +theorem yscale_step {fP : Addr} {G : Poly} {i : Nat} (hi : i < 32) {s : State} (hc : YConsts s) + (hz : ∀ l < 2, ZLanes (s.lane .xmm13 l) (fun _ => 8347681)) + (ho : ∀ l < 2, ZOdd (s.lane .xmm13 l) (s.lane .xmm12 l)) + (hdx : s.gpr .rdx = coeffAddr fP (8 * i)) (hS : YScaled s.mem fP G i) (hw : pR fP ∈ s.wr) : + WP isa (.block sbodyY) s fun s' => + YScaled s'.mem fP G (i + 1) ∧ s'.gpr .rdx = coeffAddr fP (8 * (i + 1)) ∧ + Frame [pR fP] s.mem s'.mem ∧ YConsts s' ∧ (∀ l < 2, s'.lane .xmm13 l = s.lane .xmm13 l) ∧ + (∀ l < 2, s'.lane .xmm12 l = s.lane .xmm12 l) ∧ Keep [.rdx, .rcx] s s' ∧ + s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) ∧ s'.mxcsr = s.mxcsr := by + have j0 : 8 * i + 8 ≤ 256 := by omega + have r0 : InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 0) 32 := by + rw [hdx, add_ofNat_zero]; exact f_in32 (List.mem_append_right _ hw) j0 + rw [sbodyY, List.append_assoc, List.append_assoc, WP.block_append_iff] + refine WP.mono (yld_ok r0) fun s1 ⟨L1, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (ylanes lane_vmul3 (P := fun l t => + t.xmm .xmm3 = csubV (montV (s1.lane .xmm3 l) (s1.lane .xmm13 l) (s1.lane .xmm12 l))) + fun l hl => vmul3_ok (yonly_yconsts o1 hc (by decide) (by decide) l hl)) fun s2 ⟨V2, o2⟩ => ?_ + have o12 := o1.trans o2 + have hv : ∀ l < 2, ∀ e < 4, (dword (s2.lane .xmm3 l) e).toNat = (G[8 * i + 4 * l + e]! * 8347681).val := by + intro l hl e he + have hx : DLanes (s1.lane .xmm3 l) (fun e => G[8 * i + 4 * l + e]!) := fun e he => by + rw [L1 l hl, hdx, add_ofNat_zero, dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, + hS _ (by omega), ifn (by omega)] + have hz' : ZLanes (s1.lane .xmm13 l) (fun _ => 8347681) := by + rw [o1.lane _ (by decide) l hl]; exact hz l hl + have ho' : ZOdd (s1.lane .xmm13 l) (s1.lane .xmm12 l) := by + rw [o1.lane _ (by decide) l hl, o1.lane _ (by decide) l hl]; exact ho l hl + have e2 : s2.lane .xmm3 l = _ := V2 l hl + rw [e2, dword_csubV _ he, mulZ (hx e he) (hz' e he) (dword_montV ho' (prod_lt hx hz') he)] + dsimp only + rw [Fin.mul_comm] + have w0 : InRegions s2.wr (s2.gpr .rdx) 32 := by rw [o12.wr, o12.gpr, hdx]; exact f_in32 hw j0 + vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, State.setMem_ymm, + w0, sx32] + have g2 : s2.gpr .rdx = coeffAddr fP (8 * i) := by rw [o12.gpr, hdx] + rw [g2, o12.mem] + refine ⟨fun k hk => ?_, ?_, ?_, ?_, ?_, ?_, ⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, ?_⟩ + · rw [coeffAt_write256 _ _ j0 _ hk] + split + · rename_i h + rw [ifp (show k < 8 * (i + 1) by omega), State.ymm, extract_ymm _ _ (by omega)] + split + · have := hv 0 (by decide) (k - 8 * i) (by omega) + rw [show 8 * i + 4 * 0 + (k - 8 * i) = k by omega] at this; exact this + · have := hv 1 (by decide) (k - 8 * i - 4) (by omega) + rw [show 8 * i + 4 * 1 + (k - 8 * i - 4) = k by omega] at this; exact this + · rename_i h + rw [hS k hk] + by_cases h' : k < 8 * i + · rw [ifp h', ifp (by omega)] + · rw [ifn h', ifn (by omega)] + · rw [show (32 : BitVec 64) = BitVec.ofNat 64 (4 * 8) from rfl, coeffAddr_add, + show 8 * i + 8 = 8 * (i + 1) by omega] + · exact (Frame.refl _ _).writeW (List.mem_singleton_self _) _ (pR_contains32 fP j0) + · exact ylanes_gpr (s := s2) (fun r l => by simp only [lane_setReg, lane_setFlags, State.setMem_lane]) o12 hc + (by decide) (by decide) + · intro l hl; simp only [lane_setReg, lane_setFlags, State.setMem_lane]; exact o12.lane _ (by decide) l hl + · intro l hl; simp only [lane_setReg, lane_setFlags, State.setMem_lane]; exact o12.lane _ (by decide) l hl + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr.1, hr.2, ite_false, State.setMem_gpr] + rw [o12.gpr] + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; rw [o12.rd] + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; rw [o12.wr] + · rw [o12.gpr] + · rw [o12.gpr] + · exact o12.mxcsr + +/-- Every coefficient times `8347681`. -/ +theorem yscale_ok {fP sP : Addr} (_hd : (pR sP).Disjoint (pR fP)) {G : Poly} (s : State) (hc : YConsts s) + (hdi : s.gpr .rdi = fP) (_hsi : s.gpr .rsi = sP) (hS : PolyIs s.mem fP G) (_hT : Tab zmTab s.mem sP 256) + (hwf : pR fP ∈ s.wr) (_hw : pR sP ∈ s.wr) : + WP isa yscale s fun s' => PolyIs s'.mem fP (G.map (· * 8347681)) ∧ BInvY fP s s' := by + refine WP.seq ?_ + rw [WP.block_append_iff, WP.block_append_iff] + refine WP.mono (Q := fun (w : State) => w.gpr .rdx = fP ∧ GOnly [.rdx] s w ∧ w.ymmHi = s.ymmHi) + (by vrund [hdi]; exact ⟨by gonlyd, rfl⟩) fun w ⟨hdx, og, hy⟩ => ?_ + refine WP.mono (yconst_ok .xmm13 16382 w) fun w1 ⟨l1, k1, m1, x1, o1⟩ => ?_ + refine WP.mono (ylanes (by decide) (P := fun l t => t.xmm .xmm12 = w1.lane .xmm13 l) + fun l _ => mov12_ok (w1.proj l)) fun w2 ⟨l2, o2⟩ => ?_ + have cw : YConsts w := ylanes_gpr (s := s) (og.lane hy) (YOnly.refl [] s) hc (by decide) (by decide) + have cw2 : YConsts w2 := yonly_yconsts o2 (fun l hl => + ⟨by rw [State.proj_xmm, o1 _ (by decide) l hl]; exact (cw l hl).q, + by rw [State.proj_xmm, o1 _ (by decide) l hl]; exact (cw l hl).qinv⟩) (by decide) (by decide) + have z13 : ∀ l < 2, w2.lane .xmm13 l = ofDwords 16382#32 16382#32 16382#32 16382#32 := fun l hl => by + rw [o2.lane _ (by decide) l hl, l1 l hl]; rfl + have z12 : ∀ l < 2, w2.lane .xmm12 l = ofDwords 16382#32 16382#32 16382#32 16382#32 := fun l hl => by + have e : w2.lane .xmm12 l = _ := l2 l hl + rw [e, l1 l hl]; rfl + have dx2 : w2.gpr .rdx = fP := by rw [o2.gpr, k1.gpr (by decide), hdx] + have mw2 : w2.mem = s.mem := by rw [o2.mem, m1, og.mem] + refine WP.mono (wp_rcxLoopY (N := 32) (by decide) (by decide) + (fun i u => YScaled u.mem fP G i ∧ u.gpr .rdx = coeffAddr fP (8 * i) ∧ YConsts u ∧ + (∀ l < 2, u.lane .xmm13 l = w2.lane .xmm13 l) ∧ (∀ l < 2, u.lane .xmm12 l = w2.lane .xmm12 l) ∧ + Keep [.rcx, .rdx] w2 u ∧ Frame [pR fP] w2.mem u.mem ∧ u.mxcsr = w2.mxcsr) + (fun u o hu _ => ⟨fun k hk => by + rw [o.mem, mw2, ifn (by omega)]; exact polyIs_toNat hS (by rw [n_eq]; exact hk), + by rw [o.keep.gpr (by decide), dx2, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + ylanes_gpr (s := w2) (o.lane hu) (YOnly.refl [] w2) cw2 (by decide) (by decide), + fun l _ => o.lane hu _ l, fun l _ => o.lane hu _ l, o.keep.mono (by simp), + by rw [o.mem]; exact Frame.refl _ _, o.mxcsr⟩) + (fun i hi u ⟨hS', hdx', hc', hz', hzo', hk', hf', hx'⟩ => WP.mono (yscale_step hi hc' + (fun l hl => by rw [hz' l hl, z13 l hl]; exact scale_lanes.1) + (fun l hl => by rw [hz' l hl, hzo' l hl, z13 l hl, z12 l hl]; exact scale_lanes.2) hdx' hS' + (by rw [hk'.2.2, o2.wr, k1.2.2, og.keep.2.2]; exact hwf)) + fun u' ⟨hS'', hdx'', hf'', hc'', hz'', hzo'', hk'', hcx, hzf, hx''⟩ => + ⟨⟨hS'', hdx'', hc'', fun l hl => by rw [hz'' l hl, hz' l hl], fun l hl => by rw [hzo'' l hl, hzo' l hl], + (hk'.trans hk'').mono (by simp), hf'.trans hf'', by rw [hx'', hx']⟩, hcx, hzf⟩)) + fun u ⟨hS', _, hc', _, _, hk', hf', hx'⟩ => ?_ + have kw2 : Keep [.rdx, .rax] s w2 := + ((og.keep.trans k1).trans (⟨fun r _ => by rw [o2.gpr], o2.rd, o2.wr⟩ : Keep [] w1 w2)).mono (by simp) + refine ⟨polyIs_of_toNat fun k hk => ?_, ⟨(kw2.trans hk').mono (by simp), by rw [← mw2]; exact hf', hc', + by rw [hx', o2.mxcsr, x1, og.mxcsr]⟩⟩ + rw [n_eq] at hk + rw [hS' k hk, ifp (by omega), map_mul_get _ _ (by rw [n_eq]; exact hk)] + +/-! ## The functions -/ + +/-- The code in `withMxcsr`, from its state `s1`: the prologue, then the +layers `l`, which leave `G`. -/ +theorem ynttBody_ok {t : Poly → Poly} {s s1 : State} (hs : (inPlaceK t).pre s) {l : Prog isa} {G : Poly} + (k1 : Keep [.rax, .r11] s s1) (f1 : Frame [mxR (s.gpr .rsi)] s.mem s1.mem) + (hl : ∀ s2, LIY (s.gpr .rdi) (s.gpr .rsi) s2 (polyAt s.mem (s.gpr .rdi)) s2 → s2.gpr .rdi = s.gpr .rdi → + s2.gpr .rsi = s.gpr .rsi → pR (s.gpr .rdi) ∈ s2.wr → pR (s.gpr .rsi) ∈ s2.wr → + WP isa l s2 fun s3 => PolyIs s3.mem (s.gpr .rdi) G ∧ Frame [pR (s.gpr .rdi)] s2.mem s3.mem) : + WP isa (.seq (.block ypro) l) s1 fun s' => + PolyIs s'.mem (s.gpr .rdi) G ∧ Frame [pR (s.gpr .rdi), pR (s.gpr .rsi)] s.mem s'.mem := by + have hw : pR (s.gpr .rsi) ∈ s.wr := by rw [hs.2.1]; simp + have hwf : pR (s.gpr .rdi) ∈ s.wr := by rw [hs.2.1]; simp + have hd : (pR (s.gpr .rdi)).Disjoint (pR (s.gpr .rsi)) := hs.2.2.1 + have hdi1 : s1.gpr .rdi = s.gpr .rdi := k1.gpr (by decide) + have hsi1 : s1.gpr .rsi = s.gpr .rsi := k1.gpr (by decide) + have hF1 : PolyIs s1.mem (s.gpr .rdi) (polyAt s.mem (s.gpr .rdi)) := + polyIs_frame f1 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd.sub_right (mx_sub' _)) + ⟨hs.2.2.2.2.2, rfl⟩ + refine WP.seq (WP.mono (ypro_ok hsi1 (by rw [k1.2.2]; exact hw)) fun s2 ⟨hT, hc, hf2, k2, _⟩ => ?_) + have hF2 : PolyIs s2.mem (s.gpr .rdi) (polyAt s.mem (s.gpr .rdi)) := + polyIs_frame hf2 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) hF1 + refine WP.mono (hl s2 ⟨hF2, hT, hc, Keep.refl _ _, Frame.refl _ _⟩ + (by rw [k2.gpr (by decide), hdi1]) (by rw [k2.gpr (by decide), hsi1]) + (by rw [k2.2.2, k1.2.2]; exact hwf) (by rw [k2.2.2, k1.2.2]; exact hw)) fun s3 ⟨hP, hf3⟩ => ⟨hP, ?_⟩ + refine (frame_fs f1 ?_).trans ((frame_fs hf2 ?_).trans (frame_fs hf3 ?_)) <;> + intro r hr <;> simp only [List.mem_singleton] at hr <;> subst hr + exacts [.inr (mx_sub' _), .inr fun _ h => h, .inl fun _ h => h] + +/-- `withMxcsr` around the body, and the ABI. -/ +theorem ymx_correct {t : Poly → Poly} {l : Prog isa} (s : State) (hs : (inPlaceK t).pre s) + (hk : writesOnly [.rax, .rcx, .rdx, .r8, .r9] (.seq (.block ypro) l) = true) + (hctl : ctlOk (VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 (.seq (.block ypro) l)) = true) + (hk' : writesOnly [.rax, .rcx, .rdx, .r8, .r9, .r11] + (VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 (.seq (.block ypro) l)) = true) + (hl : ∀ s1, Keep [.rax, .r11] s s1 → Frame [mxR (s.gpr .rsi)] s.mem s1.mem → + WP isa (.seq (.block ypro) l) s1 fun s' => PolyIs s'.mem (s.gpr .rdi) (t (polyAt s.mem (s.gpr .rdi))) ∧ + Frame [pR (s.gpr .rdi), pR (s.gpr .rsi)] s.mem s'.mem) : + ∃ tr s', Exec isa (VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 (.seq (.block ypro) l)) s tr s' ∧ + abiPreserved s s' ∧ (inPlaceK t).post s s' := by + have hw : pR (s.gpr .rsi) ∈ s.wr := by rw [hs.2.1]; simp + have hd : (pR (s.gpr .rdi)).Disjoint (pR (s.gpr .rsi)) := hs.2.2.1 + have hW := withMxcsr_ok (c := .seq (.block ypro) l) (by decide) [.rax, .rcx, .rdx, .r8, .r9] (by decide) rfl hw + hk (hl) + obtain ⟨tr, s', he, ⟨s2, ⟨hP, hf⟩, hf', -⟩, hk⟩ := WP.keep [.rax, .rcx, .rdx, .r8, .r9, .r11] hW hk' + refine ⟨tr, s', he, abiPreserved_of_ctl hctl he (gprPreserved_of hk (by decide) + (hf.trans (hf'.sub fun r hr => ⟨_, List.mem_cons_of_mem _ (List.mem_singleton_self _), ?_⟩)) + (by simpa using ⟨hs.2.2.2.1, hs.2.2.2.2.1⟩)), ?_⟩ + · rw [List.mem_singleton.mp hr]; exact mx_sub' _ + · exact polyIs_frame hf' (fun r hr => by + rw [List.mem_singleton.mp hr]; exact hd.sub_right (mx_sub' _)) hP + +theorem nttY_correct (s : State) (hs : (inPlaceK ntt).pre s) : + ∃ t s', Exec isa nttAvx2 s t s' ∧ abiPreserved s s' ∧ (inPlaceK ntt).post s s' := by + have hd : (pR (s.gpr .rsi)).Disjoint (pR (s.gpr .rdi)) := hs.2.2.1.symm + refine ymx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 => + WP.mono (ynttBody_ok hs k1 f1 (G := nttLens.foldl nttLayer (polyAt s.mem (s.gpr .rdi))) + fun s2 hI hdi hsi hwf hw => ?_) fun s' ⟨hP, hf⟩ => ⟨by rw [ntt_eq_layers]; exact hP, hf⟩ + simp only [nttLens, List.foldl_cons, List.foldl_nil] + refine LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 128 1 (by decide) (by decide)) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 64 2 (by decide) (by decide)) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 32 4 (by decide) (by decide)) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 16 8 (by decide) (by decide)) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay_ok hd 8 16 (by decide) (by decide)) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay4_ok hd) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay2_ok hd) ?_ hI + exact fun _ hI => LIY.seq hdi hsi hwf hw hd (yfwdLay1_ok hd) (fun _ hI => hI.epi) hI + +theorem nttInvY_correct (s : State) (hs : (inPlaceK nttInv).pre s) : + ∃ t s', Exec isa nttInvAvx2 s t s' ∧ abiPreserved s s' ∧ (inPlaceK nttInv).post s s' := by + have hd : (pR (s.gpr .rsi)).Disjoint (pR (s.gpr .rdi)) := hs.2.2.1.symm + refine ymx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 => + WP.mono (ynttBody_ok hs k1 f1 + (G := (nttInvLens.foldl nttInvLayer (polyAt s.mem (s.gpr .rdi))).map (· * 8347681)) + fun s2 hI hdi hsi hwf hw => ?_) fun s' ⟨hP, hf⟩ => ⟨by rw [nttInv_eq_layers]; exact hP, hf⟩ + simp only [nttInvLens, List.foldl_cons, List.foldl_nil] + refine LIY.seq hdi hsi hwf hw hd (yinvLay1_ok hd) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay2_ok hd) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay4_ok hd) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 8 31 (by decide) (by decide)) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 16 15 (by decide) (by decide)) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 32 7 (by decide) (by decide)) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 64 3 (by decide) (by decide)) ?_ hI + refine fun _ hI => LIY.seq hdi hsi hwf hw hd (yinvLay_ok hd 128 1 (by decide) (by decide)) ?_ hI + exact fun _ hI => LIY.seq hdi hsi hwf hw hd (yscale_ok hd) (fun _ hI => hI.epi) hI + +theorem nttY_ct : ConstantTime isa (inPlaceK ntt).pre (inPlaceK ntt).pub nttAvx2 := + VG.Taint.constantTime (A := taint) (X86_64.Taint.ofRegs [.rdi, .rsi, .rsp]) inPlace_agree (by taint_decide) + +theorem nttInvY_ct : ConstantTime isa (inPlaceK nttInv).pre (inPlaceK nttInv).pub nttInvAvx2 := + VG.Taint.constantTime (A := taint) (X86_64.Taint.ofRegs [.rdi, .rsi, .rsp]) inPlace_agree (by taint_decide) + +theorem nttY_verified : Verified X86_64.target nttAvx2 (Spec.MlDsa.nttContract X86_64.abi) := + Verified.of_correct nttY_correct nttY_ct (by + mldsa_implies [Spec.MlDsa.nttContract, Spec.MlDsa.inPlaceContract, Spec.MlDsa.inPlaceSig, inPlaceK, + X86_64.abi, X86_64.argRegs] [inPlaceSat] using inPlaceSat) + +theorem nttInvY_verified : Verified X86_64.target nttInvAvx2 (Spec.MlDsa.nttInvContract X86_64.abi) := + Verified.of_correct nttInvY_correct nttInvY_ct (by + mldsa_implies [Spec.MlDsa.nttInvContract, Spec.MlDsa.inPlaceContract, Spec.MlDsa.inPlaceSig, inPlaceK, + X86_64.abi, X86_64.argRegs] [inPlaceSat] using inPlaceSat) + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean new file mode 100644 index 000000000..5b74ee5fe --- /dev/null +++ b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean @@ -0,0 +1,18 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.BackendAvx2 + +/-! +# ML-DSA's polynomial arithmetic on x86-64: AVX2 + +A variant of `MlDsaArith` on x86-64 (see `TCB/Emit.lean`): +`vg_mldsa_ntt_avx2`, `vg_mldsa_inv_ntt_avx2`, +`vg_mldsa_multiply_ntt_avx2`, `vg_mldsa_multiply_add_ntt_avx2`, +`vg_mldsa_add_avx2` and `vg_mldsa_sub_avx2`, on eight coefficients at a time +in AVX2 registers, which need AVX and AVX2; key generation, signing and +verification calling them need them too. +-/ + +namespace VG.Variants.MlDsaArith.X86_64.Avx2 + +def variant : Proof.MlDsa.X86_64.ArithImpl := .avx2 + +end VG.Variants.MlDsaArith.X86_64.Avx2 diff --git a/src/asm/x86_64/mldsa.rs b/src/asm/x86_64/mldsa.rs index 01e804f36..541b9d25a 100644 --- a/src/asm/x86_64/mldsa.rs +++ b/src/asm/x86_64/mldsa.rs @@ -1754,6 +1754,1688 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_sub(f: *mut [u32; 256], g: *const ) } +/// The CPU features `vg_mldsa_ntt_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_NTT_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// The ML-DSA number-theoretic transform, `NTT` (FIPS 204 Algorithm 41), of the polynomial `*f` (256 coefficients less than `q` = 8380417), in place. +/// +/// Contract: `VG.Spec.MlDsa.nttContract`. Constant time: only the pointers may affect timing, not the data. +/// +/// The function computes on eight coefficients at a time in AVX2 registers, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `f` must be valid for reads and writes of 1024 bytes. +/// * `scratch` must be valid for reads and writes of 1024 bytes. +/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417. +/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `f` and `scratch` must not overlap each other (distinct Rust objects never do). +/// * Neither `f` nor `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_ntt_avx2(f: *mut [u32; 256], scratch: *mut [u64; 128]) { + core::arch::naked_asm!( + "stmxcsr DWORD PTR [rsi+768]", + "mov r11d, DWORD PTR [rsi+768]", + "and r11d, 65535", + "mov eax, 8127", + "mov DWORD PTR [rsi+772], eax", + "ldmxcsr DWORD PTR [rsi+772]", + "lfence", + "movabs r9, 111012023893504", + "mov QWORD PTR [rsi], r9", + "movabs r9, 33764919763013891", + "mov QWORD PTR [rsi+8], r9", + "movabs r9, 32652304184483396", + "mov QWORD PTR [rsi+16], r9", + "movabs r9, 2003464812134697", + "mov QWORD PTR [rsi+24], r9", + "movabs r9, 10107995079564843", + "mov QWORD PTR [rsi+32], r9", + "movabs r9, 27008953388524718", + "mov QWORD PTR [rsi+40], r9", + "movabs r9, 23603259066260085", + "mov QWORD PTR [rsi+48], r9", + "movabs r9, 11510954738022985", + "mov QWORD PTR [rsi+56], r9", + "movabs r9, 4398527550166616", + "mov QWORD PTR [rsi+64], r9", + "movabs r9, 15401443493111205", + "mov QWORD PTR [rsi+72], r9", + "movabs r9, 31185040284548497", + "mov QWORD PTR [rsi+80], r9", + "movabs r9, 26937467947448680", + "mov QWORD PTR [rsi+88], r9", + "movabs r9, 19416172761943511", + "mov QWORD PTR [rsi+96], r9", + "movabs r9, 21916122901808376", + "mov QWORD PTR [rsi+104], r9", + "movabs r9, 35910200088776757", + "mov QWORD PTR [rsi+112], r9", + "movabs r9, 1202612321726977", + "mov QWORD PTR [rsi+120], r9", + "movabs r9, 411354790447719", + "mov QWORD PTR [rsi+128], r9", + "movabs r9, 15163111458665677", + "mov QWORD PTR [rsi+136], r9", + "movabs r9, 20565458766169348", + "mov QWORD PTR [rsi+144], r9", + "movabs r9, 16816682460325845", + "mov QWORD PTR [rsi+152], r9", + "movabs r9, 22920956268049798", + "mov QWORD PTR [rsi+160], r9", + "movabs r9, 23677166863944342", + "mov QWORD PTR [rsi+168], r9", + "movabs r9, 34703121006855168", + "mov QWORD PTR [rsi+176], r9", + "movabs r9, 33677345376425628", + "mov QWORD PTR [rsi+184], r9", + "movabs r9, 28933472397947454", + "mov QWORD PTR [rsi+192], r9", + "movabs r9, 19579390106369566", + "mov QWORD PTR [rsi+200], r9", + "movabs r9, 26799599498134591", + "mov QWORD PTR [rsi+208], r9", + "movabs r9, 15889643835192413", + "mov QWORD PTR [rsi+216], r9", + "movabs r9, 2282148053410728", + "mov QWORD PTR [rsi+224], r9", + "movabs r9, 16668952760323958", + "mov QWORD PTR [rsi+232], r9", + "movabs r9, 25011900965946676", + "mov QWORD PTR [rsi+240], r9", + "movabs r9, 23977247637478740", + "mov QWORD PTR [rsi+248], r9", + "movabs r9, 29427887555995366", + "mov QWORD PTR [rsi+256], r9", + "movabs r9, 22931526182748624", + "mov QWORD PTR [rsi+264], r9", + "movabs r9, 14929091579459166", + "mov QWORD PTR [rsi+272], r9", + "movabs r9, 29185144592086471", + "mov QWORD PTR [rsi+280], r9", + "movabs r9, 8329290213797750", + "mov QWORD PTR [rsi+288], r9", + "movabs r9, 31697782066745459", + "mov QWORD PTR [rsi+296], r9", + "movabs r9, 22432987854353025", + "mov QWORD PTR [rsi+304], r9", + "movabs r9, 545125843890401", + "mov QWORD PTR [rsi+312], r9", + "movabs r9, 31769864501989618", + "mov QWORD PTR [rsi+320], r9", + "movabs r9, 11662103226140216", + "mov QWORD PTR [rsi+328], r9", + "movabs r9, 20130185304250276", + "mov QWORD PTR [rsi+336], r9", + "movabs r9, 25354781094156910", + "mov QWORD PTR [rsi+344], r9", + "movabs r9, 30717142252233347", + "mov QWORD PTR [rsi+352], r9", + "movabs r9, 30375073877558844", + "mov QWORD PTR [rsi+360], r9", + "movabs r9, 5794237307816926", + "mov QWORD PTR [rsi+368], r9", + "movabs r9, 29849966874477923", + "mov QWORD PTR [rsi+376], r9", + "movabs r9, 1137925820308458", + "mov QWORD PTR [rsi+384], r9", + "movabs r9, 13305774323778583", + "mov QWORD PTR [rsi+392], r9", + "movabs r9, 31268732009491712", + "mov QWORD PTR [rsi+400], r9", + "movabs r9, 17002134848261444", + "mov QWORD PTR [rsi+408], r9", + "movabs r9, 35956774717033419", + "mov QWORD PTR [rsi+416], r9", + "movabs r9, 22036141462600008", + "mov QWORD PTR [rsi+424], r9", + "movabs r9, 35087477629023596", + "mov QWORD PTR [rsi+432], r9", + "movabs r9, 30337763489061025", + "mov QWORD PTR [rsi+440], r9", + "movabs r9, 20732429908239468", + "mov QWORD PTR [rsi+448], r9", + "movabs r9, 28041905903253186", + "mov QWORD PTR [rsi+456], r9", + "movabs r9, 35231517950811284", + "mov QWORD PTR [rsi+464], r9", + "movabs r9, 5760968484459269", + "mov QWORD PTR [rsi+472], r9", + "movabs r9, 29186403016613413", + "mov QWORD PTR [rsi+480], r9", + "movabs r9, 29809972144732485", + "mov QWORD PTR [rsi+488], r9", + "movabs r9, 19324591173389987", + "mov QWORD PTR [rsi+496], r9", + "movabs r9, 16492506917666904", + "mov QWORD PTR [rsi+504], r9", + "movabs r9, 14635985826474643", + "mov QWORD PTR [rsi+512], r9", + "movabs r9, 16398082059229396", + "mov QWORD PTR [rsi+520], r9", + "movabs r9, 9638297459285875", + "mov QWORD PTR [rsi+528], r9", + "movabs r9, 20692959164533664", + "mov QWORD PTR [rsi+536], r9", + "movabs r9, 10455835889373941", + "mov QWORD PTR [rsi+544], r9", + "movabs r9, 15088997507073265", + "mov QWORD PTR [rsi+552], r9", + "movabs r9, 19847271512942753", + "mov QWORD PTR [rsi+560], r9", + "movabs r9, 22278162875259735", + "mov QWORD PTR [rsi+568], r9", + "movabs r9, 7984765110959710", + "mov QWORD PTR [rsi+576], r9", + "movabs r9, 3517724245221606", + "mov QWORD PTR [rsi+584], r9", + "movabs r9, 29065087369580419", + "mov QWORD PTR [rsi+592], r9", + "movabs r9, 33749496538019589", + "mov QWORD PTR [rsi+600], r9", + "movabs r9, 22582830675910690", + "mov QWORD PTR [rsi+608], r9", + "movabs r9, 13774157688799364", + "mov QWORD PTR [rsi+616], r9", + "movabs r9, 33738338209470846", + "mov QWORD PTR [rsi+624], r9", + "movabs r9, 20549610337740179", + "mov QWORD PTR [rsi+632], r9", + "movabs r9, 1232604074686745", + "mov QWORD PTR [rsi+640], r9", + "movabs r9, 17645078572608834", + "mov QWORD PTR [rsi+648], r9", + "movabs r9, 21638646536106727", + "mov QWORD PTR [rsi+656], r9", + "movabs r9, 872067341384903", + "mov QWORD PTR [rsi+664], r9", + "movabs r9, 11559822875647717", + "mov QWORD PTR [rsi+672], r9", + "movabs r9, 5433172289935931", + "mov QWORD PTR [rsi+680], r9", + "movabs r9, 5358487101890844", + "mov QWORD PTR [rsi+688], r9", + "movabs r9, 6854656137752657", + "mov QWORD PTR [rsi+696], r9", + "movabs r9, 5370830838457625", + "mov QWORD PTR [rsi+704], r9", + "movabs r9, 20753904747165841", + "mov QWORD PTR [rsi+712], r9", + "movabs r9, 8027804982718602", + "mov QWORD PTR [rsi+720], r9", + "movabs r9, 31479735164668747", + "mov QWORD PTR [rsi+728], r9", + "movabs r9, 5314055668205759", + "mov QWORD PTR [rsi+736], r9", + "movabs r9, 29850847345983039", + "mov QWORD PTR [rsi+744], r9", + "movabs r9, 5636951310400997", + "mov QWORD PTR [rsi+752], r9", + "movabs r9, 27564133741392515", + "mov QWORD PTR [rsi+760], r9", + "movabs r9, 8233800205883732", + "mov QWORD PTR [rsi+768], r9", + "movabs r9, 30088883024233849", + "mov QWORD PTR [rsi+776], r9", + "movabs r9, 3338009929245701", + "mov QWORD PTR [rsi+784], r9", + "movabs r9, 14628791756398056", + "mov QWORD PTR [rsi+792], r9", + "movabs r9, 23830870862829877", + "mov QWORD PTR [rsi+800], r9", + "movabs r9, 28061014216302585", + "mov QWORD PTR [rsi+808], r9", + "movabs r9, 19997999096164636", + "mov QWORD PTR [rsi+816], r9", + "movabs r9, 19771555530215640", + "mov QWORD PTR [rsi+824], r9", + "movabs r9, 10447056982320729", + "mov QWORD PTR [rsi+832], r9", + "movabs r9, 35286145636332471", + "mov QWORD PTR [rsi+840], r9", + "movabs r9, 14470225858518424", + "mov QWORD PTR [rsi+848], r9", + "movabs r9, 30807937853347003", + "mov QWORD PTR [rsi+856], r9", + "movabs r9, 699409659546815", + "mov QWORD PTR [rsi+864], r9", + "movabs r9, 12945035726727688", + "mov QWORD PTR [rsi+872], r9", + "movabs r9, 7098008983067813", + "mov QWORD PTR [rsi+880], r9", + "movabs r9, 28266511219523944", + "mov QWORD PTR [rsi+888], r9", + "movabs r9, 15135022374814013", + "mov QWORD PTR [rsi+896], r9", + "movabs r9, 1158610381635861", + "mov QWORD PTR [rsi+904], r9", + "movabs r9, 31802227079365879", + "mov QWORD PTR [rsi+912], r9", + "movabs r9, 2027559572878823", + "mov QWORD PTR [rsi+920], r9", + "movabs r9, 7402805639339334", + "mov QWORD PTR [rsi+928], r9", + "movabs r9, 8205002449640623", + "mov QWORD PTR [rsi+936], r9", + "movabs r9, 31250542829661849", + "mov QWORD PTR [rsi+944], r9", + "movabs r9, 19527171898598875", + "mov QWORD PTR [rsi+952], r9", + "movabs r9, 26390134497937253", + "mov QWORD PTR [rsi+960], r9", + "movabs r9, 26173917256840158", + "mov QWORD PTR [rsi+968], r9", + "movabs r9, 31797902045269139", + "mov QWORD PTR [rsi+976], r9", + "movabs r9, 20765007236602922", + "mov QWORD PTR [rsi+984], r9", + "movabs r9, 16834811519265361", + "mov QWORD PTR [rsi+992], r9", + "movabs r9, 30142973844857679", + "mov QWORD PTR [rsi+1000], r9", + "movabs r9, 6014775284471242", + "mov QWORD PTR [rsi+1008], r9", + "movabs r9, 8490214048855735", + "mov QWORD PTR [rsi+1016], r9", + "mov eax, 8380417", + "vmovq xmm15, rax", + "vpbroadcastd ymm15, xmm15", + "mov eax, -58728449", + "vmovq xmm14, rax", + "vpbroadcastd ymm14, xmm14", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 4", + "mov eax, 1", + "20:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, 4", + "mov ecx, 16", + "21:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+512]", + "vpshufd ymm4, ymm1, 245", + "vpmuludq ymm1, ymm1, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm1, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm1, ymm1, ymm2", + "vpsrlq ymm1, ymm1, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm1, ymm1, ymm4", + "vpsubd ymm1, ymm1, ymm15", + "vpsrad ymm2, ymm1, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm1, ymm1, ymm2", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpsubd ymm3, ymm3, ymm1", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+512], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 21b", + "add rdx, 512", + "sub rax, 1", + "jne 20b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 8", + "mov eax, 2", + "22:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, 4", + "mov ecx, 8", + "23:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+256]", + "vpshufd ymm4, ymm1, 245", + "vpmuludq ymm1, ymm1, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm1, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm1, ymm1, ymm2", + "vpsrlq ymm1, ymm1, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm1, ymm1, ymm4", + "vpsubd ymm1, ymm1, ymm15", + "vpsrad ymm2, ymm1, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm1, ymm1, ymm2", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpsubd ymm3, ymm3, ymm1", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+256], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 23b", + "add rdx, 256", + "sub rax, 1", + "jne 22b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 16", + "mov eax, 4", + "24:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, 4", + "mov ecx, 4", + "25:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+128]", + "vpshufd ymm4, ymm1, 245", + "vpmuludq ymm1, ymm1, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm1, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm1, ymm1, ymm2", + "vpsrlq ymm1, ymm1, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm1, ymm1, ymm4", + "vpsubd ymm1, ymm1, ymm15", + "vpsrad ymm2, ymm1, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm1, ymm1, ymm2", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpsubd ymm3, ymm3, ymm1", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+128], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 25b", + "add rdx, 128", + "sub rax, 1", + "jne 24b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 32", + "mov eax, 8", + "26:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, 4", + "mov ecx, 2", + "27:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+64]", + "vpshufd ymm4, ymm1, 245", + "vpmuludq ymm1, ymm1, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm1, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm1, ymm1, ymm2", + "vpsrlq ymm1, ymm1, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm1, ymm1, ymm4", + "vpsubd ymm1, ymm1, ymm15", + "vpsrad ymm2, ymm1, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm1, ymm1, ymm2", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpsubd ymm3, ymm3, ymm1", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+64], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 27b", + "add rdx, 64", + "sub rax, 1", + "jne 26b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 64", + "mov eax, 16", + "28:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, 4", + "mov ecx, 1", + "29:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+32]", + "vpshufd ymm4, ymm1, 245", + "vpmuludq ymm1, ymm1, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm1, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm1, ymm1, ymm2", + "vpsrlq ymm1, ymm1, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm1, ymm1, ymm4", + "vpsubd ymm1, ymm1, ymm15", + "vpsrad ymm2, ymm1, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm1, ymm1, ymm2", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpsubd ymm3, ymm3, ymm1", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+32], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 29b", + "add rdx, 32", + "sub rax, 1", + "jne 28b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 128", + "mov ecx, 16", + "210:", + "vmovdqu ymm4, YMMWORD PTR [rdx]", + "vmovdqu ymm5, YMMWORD PTR [rdx+32]", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm2, ymm13, 85", + "vpshufd ymm13, ymm13, 0", + "vpblendd ymm13, ymm13, ymm2, 240", + "vpshufd ymm12, ymm13, 245", + "add r8, 8", + "vperm2i128 ymm0, ymm4, ymm5, 32", + "vperm2i128 ymm1, ymm4, ymm5, 49", + "vpshufd ymm4, ymm1, 245", + "vpmuludq ymm1, ymm1, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm1, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm1, ymm1, ymm2", + "vpsrlq ymm1, ymm1, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm1, ymm1, ymm4", + "vpsubd ymm1, ymm1, ymm15", + "vpsrad ymm2, ymm1, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm1, ymm1, ymm2", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpsubd ymm3, ymm3, ymm1", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vperm2i128 ymm4, ymm0, ymm3, 32", + "vperm2i128 ymm5, ymm0, ymm3, 49", + "vmovdqu YMMWORD PTR [rdx], ymm4", + "vmovdqu YMMWORD PTR [rdx+32], ymm5", + "add rdx, 64", + "sub rcx, 1", + "jne 210b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 256", + "mov ecx, 16", + "211:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+32]", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm2, ymm13, 245", + "vpshufd ymm13, ymm13, 160", + "vpblendd ymm13, ymm13, ymm2, 240", + "vpshufd ymm12, ymm13, 245", + "add r8, 16", + "vmovdqa ymm2, ymm0", + "vpunpcklqdq ymm0, ymm0, ymm1", + "vpunpckhqdq ymm2, ymm2, ymm1", + "vmovdqa ymm1, ymm2", + "vpshufd ymm4, ymm1, 245", + "vpmuludq ymm1, ymm1, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm1, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm1, ymm1, ymm2", + "vpsrlq ymm1, ymm1, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm1, ymm1, ymm4", + "vpsubd ymm1, ymm1, ymm15", + "vpsrad ymm2, ymm1, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm1, ymm1, ymm2", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpsubd ymm3, ymm3, ymm1", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqa ymm1, ymm0", + "vpunpcklqdq ymm0, ymm0, ymm3", + "vpunpckhqdq ymm1, ymm1, ymm3", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+32], ymm1", + "add rdx, 64", + "sub rcx, 1", + "jne 211b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 512", + "mov ecx, 16", + "212:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm2, YMMWORD PTR [rdx+32]", + "vmovdqu ymm13, YMMWORD PTR [r8]", + "vpermq ymm13, ymm13, 216", + "vpshufd ymm12, ymm13, 245", + "add r8, 32", + "vpshufd ymm0, ymm0, 216", + "vpshufd ymm2, ymm2, 216", + "vmovdqa ymm1, ymm0", + "vpunpcklqdq ymm0, ymm0, ymm2", + "vpunpckhqdq ymm1, ymm1, ymm2", + "vpshufd ymm4, ymm1, 245", + "vpmuludq ymm1, ymm1, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm1, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm1, ymm1, ymm2", + "vpsrlq ymm1, ymm1, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm1, ymm1, ymm4", + "vpsubd ymm1, ymm1, ymm15", + "vpsrad ymm2, ymm1, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm1, ymm1, ymm2", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpsubd ymm3, ymm3, ymm1", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqa ymm1, ymm0", + "vpunpckldq ymm0, ymm0, ymm3", + "vpunpckhdq ymm1, ymm1, ymm3", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+32], ymm1", + "add rdx, 64", + "sub rcx, 1", + "jne 212b", + "vzeroupper", + "lfence", + "mov DWORD PTR [rsi+768], r11d", + "ldmxcsr DWORD PTR [rsi+768]", + "ret", + ) +} + +/// The CPU features `vg_mldsa_inv_ntt_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_INV_NTT_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// The inverse of the ML-DSA number-theoretic transform, `NTT⁻¹` (FIPS 204 Algorithm 42), of `*f` (256 coefficients less than `q` = 8380417), in place. +/// +/// Contract: `VG.Spec.MlDsa.nttInvContract`. Constant time: only the pointers may affect timing, not the data. +/// +/// The function computes on eight coefficients at a time in AVX2 registers, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `f` must be valid for reads and writes of 1024 bytes. +/// * `scratch` must be valid for reads and writes of 1024 bytes. +/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417. +/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `f` and `scratch` must not overlap each other (distinct Rust objects never do). +/// * Neither `f` nor `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_inv_ntt_avx2(f: *mut [u32; 256], scratch: *mut [u64; 128]) { + core::arch::naked_asm!( + "stmxcsr DWORD PTR [rsi+768]", + "mov r11d, DWORD PTR [rsi+768]", + "and r11d, 65535", + "mov eax, 8127", + "mov DWORD PTR [rsi+772], eax", + "ldmxcsr DWORD PTR [rsi+772]", + "lfence", + "movabs r9, 111012023893504", + "mov QWORD PTR [rsi], r9", + "movabs r9, 33764919763013891", + "mov QWORD PTR [rsi+8], r9", + "movabs r9, 32652304184483396", + "mov QWORD PTR [rsi+16], r9", + "movabs r9, 2003464812134697", + "mov QWORD PTR [rsi+24], r9", + "movabs r9, 10107995079564843", + "mov QWORD PTR [rsi+32], r9", + "movabs r9, 27008953388524718", + "mov QWORD PTR [rsi+40], r9", + "movabs r9, 23603259066260085", + "mov QWORD PTR [rsi+48], r9", + "movabs r9, 11510954738022985", + "mov QWORD PTR [rsi+56], r9", + "movabs r9, 4398527550166616", + "mov QWORD PTR [rsi+64], r9", + "movabs r9, 15401443493111205", + "mov QWORD PTR [rsi+72], r9", + "movabs r9, 31185040284548497", + "mov QWORD PTR [rsi+80], r9", + "movabs r9, 26937467947448680", + "mov QWORD PTR [rsi+88], r9", + "movabs r9, 19416172761943511", + "mov QWORD PTR [rsi+96], r9", + "movabs r9, 21916122901808376", + "mov QWORD PTR [rsi+104], r9", + "movabs r9, 35910200088776757", + "mov QWORD PTR [rsi+112], r9", + "movabs r9, 1202612321726977", + "mov QWORD PTR [rsi+120], r9", + "movabs r9, 411354790447719", + "mov QWORD PTR [rsi+128], r9", + "movabs r9, 15163111458665677", + "mov QWORD PTR [rsi+136], r9", + "movabs r9, 20565458766169348", + "mov QWORD PTR [rsi+144], r9", + "movabs r9, 16816682460325845", + "mov QWORD PTR [rsi+152], r9", + "movabs r9, 22920956268049798", + "mov QWORD PTR [rsi+160], r9", + "movabs r9, 23677166863944342", + "mov QWORD PTR [rsi+168], r9", + "movabs r9, 34703121006855168", + "mov QWORD PTR [rsi+176], r9", + "movabs r9, 33677345376425628", + "mov QWORD PTR [rsi+184], r9", + "movabs r9, 28933472397947454", + "mov QWORD PTR [rsi+192], r9", + "movabs r9, 19579390106369566", + "mov QWORD PTR [rsi+200], r9", + "movabs r9, 26799599498134591", + "mov QWORD PTR [rsi+208], r9", + "movabs r9, 15889643835192413", + "mov QWORD PTR [rsi+216], r9", + "movabs r9, 2282148053410728", + "mov QWORD PTR [rsi+224], r9", + "movabs r9, 16668952760323958", + "mov QWORD PTR [rsi+232], r9", + "movabs r9, 25011900965946676", + "mov QWORD PTR [rsi+240], r9", + "movabs r9, 23977247637478740", + "mov QWORD PTR [rsi+248], r9", + "movabs r9, 29427887555995366", + "mov QWORD PTR [rsi+256], r9", + "movabs r9, 22931526182748624", + "mov QWORD PTR [rsi+264], r9", + "movabs r9, 14929091579459166", + "mov QWORD PTR [rsi+272], r9", + "movabs r9, 29185144592086471", + "mov QWORD PTR [rsi+280], r9", + "movabs r9, 8329290213797750", + "mov QWORD PTR [rsi+288], r9", + "movabs r9, 31697782066745459", + "mov QWORD PTR [rsi+296], r9", + "movabs r9, 22432987854353025", + "mov QWORD PTR [rsi+304], r9", + "movabs r9, 545125843890401", + "mov QWORD PTR [rsi+312], r9", + "movabs r9, 31769864501989618", + "mov QWORD PTR [rsi+320], r9", + "movabs r9, 11662103226140216", + "mov QWORD PTR [rsi+328], r9", + "movabs r9, 20130185304250276", + "mov QWORD PTR [rsi+336], r9", + "movabs r9, 25354781094156910", + "mov QWORD PTR [rsi+344], r9", + "movabs r9, 30717142252233347", + "mov QWORD PTR [rsi+352], r9", + "movabs r9, 30375073877558844", + "mov QWORD PTR [rsi+360], r9", + "movabs r9, 5794237307816926", + "mov QWORD PTR [rsi+368], r9", + "movabs r9, 29849966874477923", + "mov QWORD PTR [rsi+376], r9", + "movabs r9, 1137925820308458", + "mov QWORD PTR [rsi+384], r9", + "movabs r9, 13305774323778583", + "mov QWORD PTR [rsi+392], r9", + "movabs r9, 31268732009491712", + "mov QWORD PTR [rsi+400], r9", + "movabs r9, 17002134848261444", + "mov QWORD PTR [rsi+408], r9", + "movabs r9, 35956774717033419", + "mov QWORD PTR [rsi+416], r9", + "movabs r9, 22036141462600008", + "mov QWORD PTR [rsi+424], r9", + "movabs r9, 35087477629023596", + "mov QWORD PTR [rsi+432], r9", + "movabs r9, 30337763489061025", + "mov QWORD PTR [rsi+440], r9", + "movabs r9, 20732429908239468", + "mov QWORD PTR [rsi+448], r9", + "movabs r9, 28041905903253186", + "mov QWORD PTR [rsi+456], r9", + "movabs r9, 35231517950811284", + "mov QWORD PTR [rsi+464], r9", + "movabs r9, 5760968484459269", + "mov QWORD PTR [rsi+472], r9", + "movabs r9, 29186403016613413", + "mov QWORD PTR [rsi+480], r9", + "movabs r9, 29809972144732485", + "mov QWORD PTR [rsi+488], r9", + "movabs r9, 19324591173389987", + "mov QWORD PTR [rsi+496], r9", + "movabs r9, 16492506917666904", + "mov QWORD PTR [rsi+504], r9", + "movabs r9, 14635985826474643", + "mov QWORD PTR [rsi+512], r9", + "movabs r9, 16398082059229396", + "mov QWORD PTR [rsi+520], r9", + "movabs r9, 9638297459285875", + "mov QWORD PTR [rsi+528], r9", + "movabs r9, 20692959164533664", + "mov QWORD PTR [rsi+536], r9", + "movabs r9, 10455835889373941", + "mov QWORD PTR [rsi+544], r9", + "movabs r9, 15088997507073265", + "mov QWORD PTR [rsi+552], r9", + "movabs r9, 19847271512942753", + "mov QWORD PTR [rsi+560], r9", + "movabs r9, 22278162875259735", + "mov QWORD PTR [rsi+568], r9", + "movabs r9, 7984765110959710", + "mov QWORD PTR [rsi+576], r9", + "movabs r9, 3517724245221606", + "mov QWORD PTR [rsi+584], r9", + "movabs r9, 29065087369580419", + "mov QWORD PTR [rsi+592], r9", + "movabs r9, 33749496538019589", + "mov QWORD PTR [rsi+600], r9", + "movabs r9, 22582830675910690", + "mov QWORD PTR [rsi+608], r9", + "movabs r9, 13774157688799364", + "mov QWORD PTR [rsi+616], r9", + "movabs r9, 33738338209470846", + "mov QWORD PTR [rsi+624], r9", + "movabs r9, 20549610337740179", + "mov QWORD PTR [rsi+632], r9", + "movabs r9, 1232604074686745", + "mov QWORD PTR [rsi+640], r9", + "movabs r9, 17645078572608834", + "mov QWORD PTR [rsi+648], r9", + "movabs r9, 21638646536106727", + "mov QWORD PTR [rsi+656], r9", + "movabs r9, 872067341384903", + "mov QWORD PTR [rsi+664], r9", + "movabs r9, 11559822875647717", + "mov QWORD PTR [rsi+672], r9", + "movabs r9, 5433172289935931", + "mov QWORD PTR [rsi+680], r9", + "movabs r9, 5358487101890844", + "mov QWORD PTR [rsi+688], r9", + "movabs r9, 6854656137752657", + "mov QWORD PTR [rsi+696], r9", + "movabs r9, 5370830838457625", + "mov QWORD PTR [rsi+704], r9", + "movabs r9, 20753904747165841", + "mov QWORD PTR [rsi+712], r9", + "movabs r9, 8027804982718602", + "mov QWORD PTR [rsi+720], r9", + "movabs r9, 31479735164668747", + "mov QWORD PTR [rsi+728], r9", + "movabs r9, 5314055668205759", + "mov QWORD PTR [rsi+736], r9", + "movabs r9, 29850847345983039", + "mov QWORD PTR [rsi+744], r9", + "movabs r9, 5636951310400997", + "mov QWORD PTR [rsi+752], r9", + "movabs r9, 27564133741392515", + "mov QWORD PTR [rsi+760], r9", + "movabs r9, 8233800205883732", + "mov QWORD PTR [rsi+768], r9", + "movabs r9, 30088883024233849", + "mov QWORD PTR [rsi+776], r9", + "movabs r9, 3338009929245701", + "mov QWORD PTR [rsi+784], r9", + "movabs r9, 14628791756398056", + "mov QWORD PTR [rsi+792], r9", + "movabs r9, 23830870862829877", + "mov QWORD PTR [rsi+800], r9", + "movabs r9, 28061014216302585", + "mov QWORD PTR [rsi+808], r9", + "movabs r9, 19997999096164636", + "mov QWORD PTR [rsi+816], r9", + "movabs r9, 19771555530215640", + "mov QWORD PTR [rsi+824], r9", + "movabs r9, 10447056982320729", + "mov QWORD PTR [rsi+832], r9", + "movabs r9, 35286145636332471", + "mov QWORD PTR [rsi+840], r9", + "movabs r9, 14470225858518424", + "mov QWORD PTR [rsi+848], r9", + "movabs r9, 30807937853347003", + "mov QWORD PTR [rsi+856], r9", + "movabs r9, 699409659546815", + "mov QWORD PTR [rsi+864], r9", + "movabs r9, 12945035726727688", + "mov QWORD PTR [rsi+872], r9", + "movabs r9, 7098008983067813", + "mov QWORD PTR [rsi+880], r9", + "movabs r9, 28266511219523944", + "mov QWORD PTR [rsi+888], r9", + "movabs r9, 15135022374814013", + "mov QWORD PTR [rsi+896], r9", + "movabs r9, 1158610381635861", + "mov QWORD PTR [rsi+904], r9", + "movabs r9, 31802227079365879", + "mov QWORD PTR [rsi+912], r9", + "movabs r9, 2027559572878823", + "mov QWORD PTR [rsi+920], r9", + "movabs r9, 7402805639339334", + "mov QWORD PTR [rsi+928], r9", + "movabs r9, 8205002449640623", + "mov QWORD PTR [rsi+936], r9", + "movabs r9, 31250542829661849", + "mov QWORD PTR [rsi+944], r9", + "movabs r9, 19527171898598875", + "mov QWORD PTR [rsi+952], r9", + "movabs r9, 26390134497937253", + "mov QWORD PTR [rsi+960], r9", + "movabs r9, 26173917256840158", + "mov QWORD PTR [rsi+968], r9", + "movabs r9, 31797902045269139", + "mov QWORD PTR [rsi+976], r9", + "movabs r9, 20765007236602922", + "mov QWORD PTR [rsi+984], r9", + "movabs r9, 16834811519265361", + "mov QWORD PTR [rsi+992], r9", + "movabs r9, 30142973844857679", + "mov QWORD PTR [rsi+1000], r9", + "movabs r9, 6014775284471242", + "mov QWORD PTR [rsi+1008], r9", + "movabs r9, 8490214048855735", + "mov QWORD PTR [rsi+1016], r9", + "mov eax, 8380417", + "vmovq xmm15, rax", + "vpbroadcastd ymm15, xmm15", + "mov eax, -58728449", + "vmovq xmm14, rax", + "vpbroadcastd ymm14, xmm14", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 992", + "mov ecx, 16", + "20:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm2, YMMWORD PTR [rdx+32]", + "vmovdqu ymm13, YMMWORD PTR [r8]", + "vpermq ymm13, ymm13, 39", + "vpshufd ymm13, ymm13, 177", + "vpshufd ymm12, ymm13, 245", + "add r8, -32", + "vpshufd ymm0, ymm0, 216", + "vpshufd ymm2, ymm2, 216", + "vmovdqa ymm1, ymm0", + "vpunpcklqdq ymm0, ymm0, ymm2", + "vpunpckhqdq ymm1, ymm1, ymm2", + "vpsubd ymm3, ymm1, ymm0", + "vpaddd ymm3, ymm3, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqa ymm1, ymm0", + "vpunpckldq ymm0, ymm0, ymm3", + "vpunpckhdq ymm1, ymm1, ymm3", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+32], ymm1", + "add rdx, 64", + "sub rcx, 1", + "jne 20b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 496", + "mov ecx, 16", + "21:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+32]", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm2, ymm13, 10", + "vpshufd ymm13, ymm13, 95", + "vpblendd ymm13, ymm13, ymm2, 240", + "vpshufd ymm12, ymm13, 245", + "add r8, -16", + "vmovdqa ymm2, ymm0", + "vpunpcklqdq ymm0, ymm0, ymm1", + "vpunpckhqdq ymm2, ymm2, ymm1", + "vmovdqa ymm1, ymm2", + "vpsubd ymm3, ymm1, ymm0", + "vpaddd ymm3, ymm3, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqa ymm1, ymm0", + "vpunpcklqdq ymm0, ymm0, ymm3", + "vpunpckhqdq ymm1, ymm1, ymm3", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+32], ymm1", + "add rdx, 64", + "sub rcx, 1", + "jne 21b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 248", + "mov ecx, 16", + "22:", + "vmovdqu ymm4, YMMWORD PTR [rdx]", + "vmovdqu ymm5, YMMWORD PTR [rdx+32]", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm2, ymm13, 0", + "vpshufd ymm13, ymm13, 85", + "vpblendd ymm13, ymm13, ymm2, 240", + "vpshufd ymm12, ymm13, 245", + "add r8, -8", + "vperm2i128 ymm0, ymm4, ymm5, 32", + "vperm2i128 ymm1, ymm4, ymm5, 49", + "vpsubd ymm3, ymm1, ymm0", + "vpaddd ymm3, ymm3, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vperm2i128 ymm4, ymm0, ymm3, 32", + "vperm2i128 ymm5, ymm0, ymm3, 49", + "vmovdqu YMMWORD PTR [rdx], ymm4", + "vmovdqu YMMWORD PTR [rdx+32], ymm5", + "add rdx, 64", + "sub rcx, 1", + "jne 22b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 124", + "mov eax, 16", + "23:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, -4", + "mov ecx, 1", + "24:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+32]", + "vpsubd ymm3, ymm1, ymm0", + "vpaddd ymm3, ymm3, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+32], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 24b", + "add rdx, 32", + "sub rax, 1", + "jne 23b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 60", + "mov eax, 8", + "25:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, -4", + "mov ecx, 2", + "26:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+64]", + "vpsubd ymm3, ymm1, ymm0", + "vpaddd ymm3, ymm3, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+64], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 26b", + "add rdx, 64", + "sub rax, 1", + "jne 25b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 28", + "mov eax, 4", + "27:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, -4", + "mov ecx, 4", + "28:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+128]", + "vpsubd ymm3, ymm1, ymm0", + "vpaddd ymm3, ymm3, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+128], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 28b", + "add rdx, 128", + "sub rax, 1", + "jne 27b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 12", + "mov eax, 2", + "29:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, -4", + "mov ecx, 8", + "210:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+256]", + "vpsubd ymm3, ymm1, ymm0", + "vpaddd ymm3, ymm3, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+256], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 210b", + "add rdx, 256", + "sub rax, 1", + "jne 29b", + "mov rdx, rdi", + "mov r8, rsi", + "add r8, 4", + "mov eax, 1", + "211:", + "vbroadcasti128 ymm13, XMMWORD PTR [r8]", + "vpshufd ymm13, ymm13, 0", + "vpshufd ymm12, ymm13, 245", + "add r8, -4", + "mov ecx, 16", + "212:", + "vmovdqu ymm0, YMMWORD PTR [rdx]", + "vmovdqu ymm1, YMMWORD PTR [rdx+512]", + "vpsubd ymm3, ymm1, ymm0", + "vpaddd ymm3, ymm3, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm0", + "vmovdqu YMMWORD PTR [rdx+512], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 212b", + "add rdx, 512", + "sub rax, 1", + "jne 211b", + "mov rdx, rdi", + "mov eax, 16382", + "vmovq xmm13, rax", + "vpbroadcastd ymm13, xmm13", + "vmovdqa ymm12, ymm13", + "mov ecx, 32", + "213:", + "vmovdqu ymm3, YMMWORD PTR [rdx]", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdx], ymm3", + "add rdx, 32", + "sub rcx, 1", + "jne 213b", + "vzeroupper", + "lfence", + "mov DWORD PTR [rsi+768], r11d", + "ldmxcsr DWORD PTR [rsi+768]", + "ret", + ) +} + +/// The CPU features `vg_mldsa_multiply_ntt_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_MULTIPLY_NTT_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// The product of two elements of `T_q`, `MultiplyNTT` (FIPS 204 Algorithm 45): writes the coefficientwise product of `*f` and `*g` modulo `q` = 8380417 to `*h`. +/// +/// Contract: `VG.Spec.MlDsa.mulContract`. Constant time: only the pointers may affect timing, not the data. +/// +/// The function computes on eight coefficients at a time in AVX2 registers. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `h` must be valid for reads and writes of 1024 bytes. +/// * `f` must be valid for reads of 1024 bytes. +/// * `g` must be valid for reads of 1024 bytes. +/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417. +/// * Each of the 256 `u32`s of `g` must be less than `q` = 8380417. +/// * `h` must not overlap `f` or `g` (distinct Rust objects never do). +/// * None of `h`, `f` and `g` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_ntt_avx2(h: *mut [u32; 256], f: *const [u32; 256], g: *const [u32; 256]) { + core::arch::naked_asm!( + "mov r8, rdi", + "vmovdqu ymm6, YMMWORD PTR [rdi+992]", + "stmxcsr DWORD PTR [r8+1016]", + "mov r11d, DWORD PTR [r8+1016]", + "and r11d, 65535", + "mov eax, 8127", + "mov DWORD PTR [r8+1020], eax", + "ldmxcsr DWORD PTR [r8+1020]", + "lfence", + "mov eax, 8380417", + "vmovq xmm15, rax", + "vpbroadcastd ymm15, xmm15", + "mov eax, -58728449", + "vmovq xmm14, rax", + "vpbroadcastd ymm14, xmm14", + "mov eax, 2365951", + "vmovq xmm11, rax", + "vpbroadcastd ymm11, xmm11", + "mov ecx, 31", + "20:", + "vmovdqu ymm3, YMMWORD PTR [rsi]", + "vmovdqu ymm13, YMMWORD PTR [rdx]", + "vmovdqu ymm5, YMMWORD PTR [rdi]", + "vpshufd ymm12, ymm13, 245", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm11", + "vpmuludq ymm4, ymm4, ymm11", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdi], ymm3", + "add rdi, 32", + "add rsi, 32", + "add rdx, 32", + "sub rcx, 1", + "jne 20b", + "vmovdqu ymm3, YMMWORD PTR [rsi]", + "vmovdqu ymm13, YMMWORD PTR [rdx]", + "vmovdqa ymm5, ymm6", + "vpshufd ymm12, ymm13, 245", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm11", + "vpmuludq ymm4, ymm4, ymm11", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "lfence", + "mov DWORD PTR [r8+1016], r11d", + "ldmxcsr DWORD PTR [r8+1016]", + "vmovdqu YMMWORD PTR [rdi], ymm3", + "vzeroupper", + "ret", + ) +} + +/// The CPU features `vg_mldsa_multiply_add_ntt_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_MULTIPLY_ADD_NTT_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// Adds the product of two elements of `T_q` to a third, `AddNTT(h, MultiplyNTT(f, g))` (FIPS 204 Algorithms 44 and 45): adds the coefficientwise product of `*f` and `*g` to `*h`, modulo `q` = 8380417. +/// +/// Contract: `VG.Spec.MlDsa.mulAddContract`. Constant time: only the pointers may affect timing, not the data. +/// +/// The function computes on eight coefficients at a time in AVX2 registers. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing), through the last 8 bytes of `h`, which it stores last, and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `h` must be valid for reads and writes of 1024 bytes. +/// * `f` must be valid for reads of 1024 bytes. +/// * `g` must be valid for reads of 1024 bytes. +/// * Each of the 256 `u32`s of `h` must be less than `q` = 8380417. +/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417. +/// * Each of the 256 `u32`s of `g` must be less than `q` = 8380417. +/// * `h` must not overlap `f` or `g` (distinct Rust objects never do). +/// * None of `h`, `f` and `g` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_multiply_add_ntt_avx2(h: *mut [u32; 256], f: *const [u32; 256], g: *const [u32; 256]) { + core::arch::naked_asm!( + "mov r8, rdi", + "vmovdqu ymm6, YMMWORD PTR [rdi+992]", + "stmxcsr DWORD PTR [r8+1016]", + "mov r11d, DWORD PTR [r8+1016]", + "and r11d, 65535", + "mov eax, 8127", + "mov DWORD PTR [r8+1020], eax", + "ldmxcsr DWORD PTR [r8+1020]", + "lfence", + "mov eax, 8380417", + "vmovq xmm15, rax", + "vpbroadcastd ymm15, xmm15", + "mov eax, -58728449", + "vmovq xmm14, rax", + "vpbroadcastd ymm14, xmm14", + "mov eax, 2365951", + "vmovq xmm11, rax", + "vpbroadcastd ymm11, xmm11", + "mov ecx, 31", + "20:", + "vmovdqu ymm3, YMMWORD PTR [rsi]", + "vmovdqu ymm13, YMMWORD PTR [rdx]", + "vmovdqu ymm5, YMMWORD PTR [rdi]", + "vpshufd ymm12, ymm13, 245", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm11", + "vpmuludq ymm4, ymm4, ymm11", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vpaddd ymm3, ymm3, ymm5", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vmovdqu YMMWORD PTR [rdi], ymm3", + "add rdi, 32", + "add rsi, 32", + "add rdx, 32", + "sub rcx, 1", + "jne 20b", + "vmovdqu ymm3, YMMWORD PTR [rsi]", + "vmovdqu ymm13, YMMWORD PTR [rdx]", + "vmovdqa ymm5, ymm6", + "vpshufd ymm12, ymm13, 245", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm13", + "vpmuludq ymm4, ymm4, ymm12", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpshufd ymm4, ymm3, 245", + "vpmuludq ymm3, ymm3, ymm11", + "vpmuludq ymm4, ymm4, ymm11", + "vpmuludq ymm2, ymm3, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm3, ymm3, ymm2", + "vpsrlq ymm3, ymm3, 32", + "vpmuludq ymm2, ymm4, ymm14", + "vpmuludq ymm2, ymm2, ymm15", + "vpaddq ymm4, ymm4, ymm2", + "vpor ymm3, ymm3, ymm4", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "vpaddd ymm3, ymm3, ymm5", + "vpsubd ymm3, ymm3, ymm15", + "vpsrad ymm2, ymm3, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm3, ymm3, ymm2", + "lfence", + "mov DWORD PTR [r8+1016], r11d", + "ldmxcsr DWORD PTR [r8+1016]", + "vmovdqu YMMWORD PTR [rdi], ymm3", + "vzeroupper", + "ret", + ) +} + +/// The CPU features `vg_mldsa_add_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_ADD_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// Adds the polynomial `*g` to `*f` modulo `q` = 8380417, coefficient by coefficient (FIPS 204 Algorithm 44). +/// +/// Contract: `VG.Spec.MlDsa.addContract`. Constant time: only the pointers may affect timing, not the data. +/// +/// The function computes on eight coefficients at a time in AVX2 registers. +/// +/// # Safety +/// +/// * `f` must be valid for reads and writes of 1024 bytes. +/// * `g` must be valid for reads of 1024 bytes. +/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417. +/// * Each of the 256 `u32`s of `g` must be less than `q` = 8380417. +/// * `f` must not overlap `g` (distinct Rust objects never do). +/// * Neither `f` nor `g` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_add_avx2(f: *mut [u32; 256], g: *const [u32; 256]) { + core::arch::naked_asm!( + "mov eax, 8380417", + "vmovq xmm15, rax", + "vpbroadcastd ymm15, xmm15", + "mov ecx, 32", + "20:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm1, YMMWORD PTR [rsi]", + "vpaddd ymm0, ymm0, ymm1", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vmovdqu YMMWORD PTR [rdi], ymm0", + "add rdi, 32", + "add rsi, 32", + "sub rcx, 1", + "jne 20b", + "vzeroupper", + "ret", + ) +} + +/// The CPU features `vg_mldsa_sub_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_SUB_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// Subtracts the polynomial `*g` from `*f` modulo `q` = 8380417, coefficient by coefficient. +/// +/// Contract: `VG.Spec.MlDsa.subContract`. Constant time: only the pointers may affect timing, not the data. +/// +/// The function computes on eight coefficients at a time in AVX2 registers. +/// +/// # Safety +/// +/// * `f` must be valid for reads and writes of 1024 bytes. +/// * `g` must be valid for reads of 1024 bytes. +/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417. +/// * Each of the 256 `u32`s of `g` must be less than `q` = 8380417. +/// * `f` must not overlap `g` (distinct Rust objects never do). +/// * Neither `f` nor `g` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_sub_avx2(f: *mut [u32; 256], g: *const [u32; 256]) { + core::arch::naked_asm!( + "mov eax, 8380417", + "vmovq xmm15, rax", + "vpbroadcastd ymm15, xmm15", + "mov ecx, 32", + "20:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm1, YMMWORD PTR [rsi]", + "vpsubd ymm0, ymm0, ymm1", + "vpsrad ymm2, ymm0, 31", + "vpand ymm2, ymm2, ymm15", + "vpaddd ymm0, ymm0, ymm2", + "vmovdqu YMMWORD PTR [rdi], ymm0", + "add rdi, 32", + "add rsi, 32", + "sub rcx, 1", + "jne 20b", + "vzeroupper", + "ret", + ) +} + /// `SimpleBitPack(f, b)` (FIPS 204 Algorithm 16): writes the 256 coefficients of `*f` to `out` as `bitlen b`-bit little-endian fields. /// /// Contract: `VG.Spec.MlDsa.simpleBitPackContract`. Constant time: only the pointers, `b` and `len` may affect timing, not the data. diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs index 63ea03f6e..90e8d05f4 100644 --- a/src/asm/x86_64/mldsa44.rs +++ b/src/asm/x86_64/mldsa44.rs @@ -2,6 +2,3460 @@ //! Verified `mldsa44` functions for `x86_64`. #![allow(dead_code)] +/// The CPU features `vg_mldsa44_keygen_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA44_KEYGEN_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-44 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address. +/// +/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails. +/// +/// # Safety +/// +/// * `seed` must be valid for reads of 32 bytes. +/// * `pk` must be valid for reads and writes of 1312 bytes. +/// * `sk` must be valid for reads and writes of 2560 bytes. +/// * `scratch` must be valid for reads and writes of 77824 bytes. +/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do). +/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa44_keygen_avx2(seed: *const [u8; 32], pk: *mut [u8; 1312], sk: *mut [u8; 2560], scratch: *mut [u64; 9728]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+840], rbx", + "mov QWORD PTR [rcx+848], rbp", + "mov QWORD PTR [rcx+856], r12", + "mov QWORD PTR [rcx+864], r13", + "mov QWORD PTR [rcx+872], r14", + "mov QWORD PTR [rcx+880], r15", + "mov rbx, rcx", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r15d, 1", + "mov eax, 4", + "mov BYTE PTR [rbx+896], al", + "mov eax, 4", + "mov BYTE PTR [rbx+897], al", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, rbx", + "add rcx, 896", + "mov r8d, 2", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 34", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 128", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", + "mov rdi, rbx", + "add rdi, 1216", + "mov rsi, rbx", + "add rsi, 1056", + "mov ecx, 64", + "21:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 21b", + "mov eax, 0", + "mov BYTE PTR [rbx+1281], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 4096", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 4096", + "mov ecx, 256", + "22:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 22b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 5120", + "mov ecx, 256", + "23:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 23b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 6144", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov ecx, 256", + "24:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 24b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 7168", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 7168", + "mov ecx, 256", + "25:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 25b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 8192", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 8192", + "mov ecx, 256", + "26:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 26b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 9216", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov ecx, 256", + "27:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 27b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 10240", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 10240", + "mov ecx, 256", + "28:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 28b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 11264", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 11264", + "mov ecx, 256", + "29:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 29b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 12288", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 12288", + "mov ecx, 256", + "210:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 210b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 13312", + "mov ecx, 256", + "211:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 211b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 14336", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 14336", + "mov ecx, 256", + "212:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 212b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 15360", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 15360", + "mov ecx, 256", + "213:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 213b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 16384", + "mov ecx, 256", + "214:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 214b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 17408", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 17408", + "mov ecx, 256", + "215:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 215b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 18432", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 18432", + "mov ecx, 256", + "216:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 216b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 19456", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 19456", + "mov ecx, 256", + "217:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 217b", + "mov eax, 0", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 20480", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 20480", + "mov ecx, 256", + "218:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 218b", + "mov eax, 1", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 21504", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 21504", + "mov ecx, 256", + "219:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 219b", + "mov eax, 2", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 22528", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 22528", + "mov ecx, 256", + "220:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 220b", + "mov eax, 3", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 23552", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 23552", + "mov ecx, 256", + "221:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 221b", + "mov eax, 4", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 24576", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 24576", + "mov ecx, 256", + "222:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 222b", + "mov eax, 5", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 25600", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 25600", + "mov ecx, 256", + "223:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 223b", + "mov eax, 6", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 26624", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 26624", + "mov ecx, 256", + "224:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 224b", + "mov eax, 7", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 27648", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 27648", + "mov ecx, 256", + "225:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 225b", + "mov rdi, r12", + "add rdi, 0", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "226:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 226b", + "mov rdi, r13", + "add rdi, 0", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "227:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 227b", + "mov rdi, r13", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 1120", + "mov ecx, 32", + "228:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 228b", + "mov rdi, rbx", + "add rdi, 20480", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 128", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 21504", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 224", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 22528", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 320", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 23552", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 416", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 24576", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 512", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 25600", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 608", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 704", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 800", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 20480", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 4096", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 6144", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 7168", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 29696", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 32", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 30720", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 896", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 8192", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 9216", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 10240", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 11264", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 29696", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 352", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 30720", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1312", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 12288", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 14336", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 15360", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 26624", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 29696", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 672", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 30720", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1728", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 17408", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 18432", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 19456", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 27648", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 29696", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 992", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 30720", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 2144", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 1312", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 88", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r13", + "add rcx, 64", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, + vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, + vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + ) +} + +/// The CPU features `vg_mldsa44_sign_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA44_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-44 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address. +/// +/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing. +/// +/// # Safety +/// +/// * `sk` must be valid for reads of 2560 bytes. +/// * `mu` must be valid for reads of 64 bytes. +/// * `rnd` must be valid for reads of 32 bytes. +/// * `sig` must be valid for reads and writes of 2420 bytes. +/// * `scratch` must be valid for reads and writes of 77824 bytes. +/// * `sk` must have been written by `vg_mldsa44_keygen`. +/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do). +/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 2420], scratch: *mut [u64; 9728]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [r8+840], rbx", + "mov QWORD PTR [r8+848], rbp", + "mov QWORD PTR [r8+856], r12", + "mov QWORD PTR [r8+864], r13", + "mov QWORD PTR [r8+872], r14", + "mov QWORD PTR [r8+880], r15", + "mov rbx, r8", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r14, rcx", + "mov r15d, 1", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 41984", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 43008", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 44032", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 49152", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 51200", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 52224", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "test r15d, r15d", + "jne 21f", + "jmp 22f", + "21:", + "mov rdi, rbp", + "add rdi, 128", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 26624", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 224", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 27648", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 320", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 28672", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 416", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 29696", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 512", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 30720", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 608", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 31744", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 704", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 32768", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 800", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 33792", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 896", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 34816", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1312", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 35840", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1728", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 36864", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 2144", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 37888", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 32", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, r13", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 128", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 960", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov eax, 0", + "mov QWORD PTR [rbx+896], rax", + "mov eax, 814", + "mov QWORD PTR [rbx+888], rax", + "23:", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 0", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 131072", + "mov rdx, rbx", + "add rdx, 14336", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 18432", + "mov rsi, rbx", + "add rsi, 14336", + "mov ecx, 1024", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 24b", + "mov rdi, rbx", + "add rdi, 18432", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 131072", + "mov rdx, rbx", + "add rdx, 15360", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 15360", + "mov ecx, 1024", + "25:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 25b", + "mov rdi, rbx", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 131072", + "mov rdx, rbx", + "add rdx, 16384", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 20480", + "mov rsi, rbx", + "add rsi, 16384", + "mov ecx, 1024", + "26:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 26b", + "mov rdi, rbx", + "add rdi, 20480", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 131072", + "mov rdx, rbx", + "add rdx, 17408", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 17408", + "mov ecx, 1024", + "27:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 27b", + "mov rdi, rbx", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 41984", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 43008", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 44032", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 49152", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 51200", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 52224", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov esi, 95232", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 43", + "mov rdx, rbx", + "add rdx, 2048", + "mov ecx, 192", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 23552", + "mov esi, 95232", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 43", + "mov rdx, rbx", + "add rdx, 2240", + "mov ecx, 192", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 24576", + "mov esi, 95232", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 43", + "mov rdx, rbx", + "add rdx, 2432", + "mov ecx, 192", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 25600", + "mov esi, 95232", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 43", + "mov rdx, rbx", + "add rdx, 2624", + "mov ecx, 192", + "call {vg_mldsa_simple_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, rbx", + "add rcx, 2048", + "mov r8d, 768", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 16", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1040", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rdi, rbx", + "add rdi, 1040", + "mov esi, 32", + "mov edx, 39", + "mov rcx, rbx", + "add rcx, 5120", + "mov r8, rbx", + "add r8, 3072", + "call {vg_mldsa_sample_in_ball}", + "test eax, eax", + "jne 28f", + "mov r15d, 0", + "mov eax, 1", + "mov QWORD PTR [rbx+888], rax", + "jmp 29f", + "28:", + "mov rdi, rbx", + "add rdi, 5120", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov r15d, 1", + "mov eax, 0", + "mov QWORD PTR [rbx+904], rax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 14336", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 14336", + "mov esi, 130994", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 15360", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 15360", + "mov esi, 130994", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 16384", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 16384", + "mov esi, 130994", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 17408", + "mov esi, 130994", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov esi, 95232", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 95154", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov esi, 95232", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 95154", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 32768", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 24576", + "mov esi, 95232", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 95154", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 33792", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov esi, 95232", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 95154", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 34816", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 95232", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 22528", + "mov ecx, 1024", + "210:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 210b", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 22528", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 22528", + "mov edx, 95232", + "mov rcx, rbx", + "add rcx, 10240", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 35840", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 95232", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 23552", + "mov ecx, 1024", + "211:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 211b", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 23552", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 23552", + "mov edx, 95232", + "mov rcx, rbx", + "add rcx, 11264", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 36864", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 95232", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 24576", + "mov ecx, 1024", + "212:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 212b", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 24576", + "mov edx, 95232", + "mov rcx, rbx", + "add rcx, 12288", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 37888", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 95232", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 25600", + "mov ecx, 1024", + "213:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 213b", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 25600", + "mov edx, 95232", + "mov rcx, rbx", + "add rcx, 13312", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov eax, DWORD PTR [rbx+904]", + "sub rax, 81", + "shr rax, 63", + "and r15d, eax", + "test r15d, r15d", + "jne 214f", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 4", + "mov QWORD PTR [rbx+896], rax", + "jmp 215f", + "214:", + "mov eax, 1", + "mov QWORD PTR [rbx+888], rax", + "215:", + "29:", + "mov rax, QWORD PTR [rbx+888]", + "sub rax, 1", + "mov QWORD PTR [rbx+888], rax", + "jne 23b", + "test r15d, r15d", + "jne 216f", + "jmp 217f", + "216:", + "mov rdi, r14", + "add rdi, 0", + "mov rsi, rbx", + "add rsi, 1040", + "mov ecx, 32", + "218:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 218b", + "mov rdi, rbx", + "add rdi, 14336", + "mov esi, 131071", + "mov edx, 131072", + "mov rcx, r14", + "add rcx, 32", + "mov r8d, 576", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 15360", + "mov esi, 131071", + "mov edx, 131072", + "mov rcx, r14", + "add rcx, 608", + "mov r8d, 576", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 16384", + "mov esi, 131071", + "mov edx, 131072", + "mov rcx, r14", + "add rcx, 1184", + "mov r8d, 576", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 17408", + "mov esi, 131071", + "mov edx, 131072", + "mov rcx, r14", + "add rcx, 1760", + "mov r8d, 576", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 10240", + "mov esi, 1024", + "mov edx, 80", + "mov rcx, r14", + "add rcx, 2336", + "mov r8d, 84", + "call {vg_mldsa_hint_bit_pack}", + "217:", + "22:", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_high_bits = sym super::mldsa::vg_mldsa_high_bits, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, + vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, + vg_mldsa_low_bits = sym super::mldsa::vg_mldsa_low_bits, + vg_mldsa_make_hint = sym super::mldsa::vg_mldsa_make_hint, + vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, + vg_mldsa_hint_bit_pack = sym super::mldsa::vg_mldsa_hint_bit_pack, + ) +} + +/// The CPU features `vg_mldsa44_verify_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA44_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-44 verification of a signature of a message representative, `ML-DSA.Verify_internal(pk, M′, σ)` (FIPS 204 Algorithm 8) with `μ` computed by the caller: with the public key `*pk`, the 64-byte message representative `μ = H(H(pk, 64) ‖ M′, 64)` at `mu` and the signature `*sig`, returns 1 if the signature is valid, and 0 if it is not or if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less). +/// +/// Contract: `VG.Spec.MlDsa.verifyContract`. Not constant time: timing may depend on the public key, the message representative and the signature. +/// +/// It calls the `vg_mldsa_*` primitives and the SHAKE256 sponge. The samplers' results are combined without a branch, so the only branches depend on the public key and the signature. +/// +/// # Safety +/// +/// * `pk` must be valid for reads of 1312 bytes. +/// * `mu` must be valid for reads of 64 bytes. +/// * `sig` must be valid for reads of 2420 bytes. +/// * `scratch` must be valid for reads and writes of 77824 bytes. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312], mu: *const [u8; 64], sig: *const [u8; 2420], scratch: *mut [u64; 9728]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+840], rbx", + "mov QWORD PTR [rcx+848], rbp", + "mov QWORD PTR [rcx+856], r12", + "mov QWORD PTR [rcx+864], r13", + "mov QWORD PTR [rcx+872], r14", + "mov QWORD PTR [rcx+880], r15", + "mov rbx, rcx", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r15d, 1", + "mov rdi, r13", + "add rdi, 2336", + "mov esi, 84", + "mov edx, 80", + "mov rcx, rbx", + "add rcx, 8192", + "mov r8d, 1024", + "call {vg_mldsa_hint_bit_unpack}", + "mov r15d, eax", + "test r15d, r15d", + "jne 20f", + "jmp 21f", + "20:", + "mov rdi, r13", + "add rdi, 32", + "mov esi, 576", + "mov edx, 131071", + "mov ecx, 131072", + "mov r8, rbx", + "add r8, 16384", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 16384", + "mov esi, 130994", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 608", + "mov esi, 576", + "mov edx, 131071", + "mov ecx, 131072", + "mov r8, rbx", + "add r8, 17408", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 17408", + "mov esi, 130994", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 1184", + "mov esi, 576", + "mov edx, 131071", + "mov ecx, 131072", + "mov r8, rbx", + "add r8, 18432", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 18432", + "mov esi, 130994", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 1760", + "mov esi, 576", + "mov edx, 131071", + "mov ecx, 131072", + "mov r8, rbx", + "add r8, 19456", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 19456", + "mov esi, 130994", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "test r15d, r15d", + "jne 22f", + "jmp 23f", + "22:", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 24b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 28672", + "mov ecx, 256", + "25:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 25b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 29696", + "mov ecx, 256", + "26:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 26b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 30720", + "mov ecx, 256", + "27:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 27b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 31744", + "mov ecx, 256", + "28:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 28b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 36864", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 36864", + "mov ecx, 256", + "29:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 29b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 37888", + "mov ecx, 256", + "210:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 210b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 38912", + "mov ecx, 256", + "211:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 211b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 39936", + "mov ecx, 256", + "212:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 45056", + "mov ecx, 256", + "213:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 213b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 46080", + "mov ecx, 256", + "214:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 214b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 47104", + "mov ecx, 256", + "215:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 215b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 48128", + "mov ecx, 256", + "216:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 53248", + "mov ecx, 256", + "217:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 217b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 54272", + "mov ecx, 256", + "218:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 218b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 55296", + "mov ecx, 256", + "219:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 219b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 56320", + "mov ecx, 256", + "220:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 220b", + "mov rdi, r13", + "add rdi, 0", + "mov esi, 32", + "mov edx, 39", + "mov rcx, rbx", + "add rcx, 23552", + "mov r8, rbx", + "add r8, 4096", + "call {vg_mldsa_sample_in_ball}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 23552", + "mov ecx, 256", + "221:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 221b", + "mov rdi, rbx", + "add rdi, 16384", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 18432", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 95232", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 43", + "mov rdx, rbx", + "add rdx, 1024", + "mov ecx, 192", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 36864", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 352", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 95232", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 43", + "mov rdx, rbx", + "add rdx, 1216", + "mov ecx, 192", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 672", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 10240", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 95232", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 43", + "mov rdx, rbx", + "add rdx, 1408", + "mov ecx, 192", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 992", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 11264", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 95232", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 43", + "mov rdx, rbx", + "add rdx, 1600", + "mov ecx, 192", + "call {vg_mldsa_simple_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 768", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 16", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 2048", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rsi, rbx", + "add rsi, 2048", + "mov rdi, r13", + "add rdi, 0", + "mov ecx, 32", + "mov edx, 0", + "222:", + "movzx eax, BYTE PTR [rsi]", + "movzx r8d, BYTE PTR [rdi]", + "xor rax, r8", + "or rdx, rax", + "add rsi, 1", + "add rdi, 1", + "sub rcx, 1", + "jne 222b", + "sub rdx, 1", + "sbb rax, rax", + "and r15d, eax", + "23:", + "21:", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, + vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, + vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1, + vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + ) +} + /// ML-DSA-44 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. /// /// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs index 6ff94ded7..6452825aa 100644 --- a/src/asm/x86_64/mldsa65.rs +++ b/src/asm/x86_64/mldsa65.rs @@ -2,6 +2,5175 @@ //! Verified `mldsa65` functions for `x86_64`. #![allow(dead_code)] +/// The CPU features `vg_mldsa65_keygen_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA65_KEYGEN_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-65 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address. +/// +/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails. +/// +/// # Safety +/// +/// * `seed` must be valid for reads of 32 bytes. +/// * `pk` must be valid for reads and writes of 1952 bytes. +/// * `sk` must be valid for reads and writes of 4032 bytes. +/// * `scratch` must be valid for reads and writes of 103424 bytes. +/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do). +/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa65_keygen_avx2(seed: *const [u8; 32], pk: *mut [u8; 1952], sk: *mut [u8; 4032], scratch: *mut [u64; 12928]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+840], rbx", + "mov QWORD PTR [rcx+848], rbp", + "mov QWORD PTR [rcx+856], r12", + "mov QWORD PTR [rcx+864], r13", + "mov QWORD PTR [rcx+872], r14", + "mov QWORD PTR [rcx+880], r15", + "mov rbx, rcx", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r15d, 1", + "mov eax, 6", + "mov BYTE PTR [rbx+896], al", + "mov eax, 5", + "mov BYTE PTR [rbx+897], al", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, rbx", + "add rcx, 896", + "mov r8d, 2", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 34", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 128", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", + "mov rdi, rbx", + "add rdi, 1216", + "mov rsi, rbx", + "add rsi, 1056", + "mov ecx, 64", + "21:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 21b", + "mov eax, 0", + "mov BYTE PTR [rbx+1281], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 4096", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 4096", + "mov ecx, 256", + "22:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 22b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 5120", + "mov ecx, 256", + "23:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 23b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 6144", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov ecx, 256", + "24:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 24b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 7168", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 7168", + "mov ecx, 256", + "25:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 25b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 8192", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 8192", + "mov ecx, 256", + "26:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 26b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 9216", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov ecx, 256", + "27:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 27b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 10240", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 10240", + "mov ecx, 256", + "28:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 28b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 11264", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 11264", + "mov ecx, 256", + "29:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 29b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 12288", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 12288", + "mov ecx, 256", + "210:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 210b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 13312", + "mov ecx, 256", + "211:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 211b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 14336", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 14336", + "mov ecx, 256", + "212:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 212b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 15360", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 15360", + "mov ecx, 256", + "213:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 213b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 16384", + "mov ecx, 256", + "214:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 214b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 17408", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 17408", + "mov ecx, 256", + "215:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 215b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 18432", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 18432", + "mov ecx, 256", + "216:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 19456", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 19456", + "mov ecx, 256", + "217:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 217b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 20480", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 20480", + "mov ecx, 256", + "218:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 218b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 21504", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 21504", + "mov ecx, 256", + "219:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 219b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 22528", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 22528", + "mov ecx, 256", + "220:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 220b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 23552", + "mov ecx, 256", + "221:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 221b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 24576", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 24576", + "mov ecx, 256", + "222:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 222b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 25600", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 25600", + "mov ecx, 256", + "223:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 223b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 26624", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 26624", + "mov ecx, 256", + "224:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 224b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 27648", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 27648", + "mov ecx, 256", + "225:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 225b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 28672", + "mov ecx, 256", + "226:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 226b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 29696", + "mov ecx, 256", + "227:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 227b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 30720", + "mov ecx, 256", + "228:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 228b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 31744", + "mov ecx, 256", + "229:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 229b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 32768", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 32768", + "mov ecx, 256", + "230:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 230b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 33792", + "mov ecx, 256", + "231:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 231b", + "mov eax, 0", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 34816", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 34816", + "mov ecx, 256", + "232:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 232b", + "mov eax, 1", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 35840", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 35840", + "mov ecx, 256", + "233:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 233b", + "mov eax, 2", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 36864", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 36864", + "mov ecx, 256", + "234:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 234b", + "mov eax, 3", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 37888", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 37888", + "mov ecx, 256", + "235:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 235b", + "mov eax, 4", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 38912", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 38912", + "mov ecx, 256", + "236:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 236b", + "mov eax, 5", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 39936", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 39936", + "mov ecx, 256", + "237:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 237b", + "mov eax, 6", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 40960", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 40960", + "mov ecx, 256", + "238:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 238b", + "mov eax, 7", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 41984", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 41984", + "mov ecx, 256", + "239:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 239b", + "mov eax, 8", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 43008", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 43008", + "mov ecx, 256", + "240:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 240b", + "mov eax, 9", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 44032", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 44032", + "mov ecx, 256", + "241:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 241b", + "mov eax, 10", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 4", + "mov rdx, rbx", + "add rdx, 45056", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 45056", + "mov ecx, 256", + "242:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 242b", + "mov rdi, r12", + "add rdi, 0", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "243:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 243b", + "mov rdi, r13", + "add rdi, 0", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "244:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 244b", + "mov rdi, r13", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 1120", + "mov ecx, 32", + "245:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 245b", + "mov rdi, rbx", + "add rdi, 34816", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 128", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 35840", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 256", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 36864", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 384", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 37888", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 512", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 38912", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 640", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 39936", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 768", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 40960", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 896", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 41984", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 1024", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 43008", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 1152", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 44032", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 1280", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 45056", + "mov esi, 4", + "mov edx, 4", + "mov rcx, r13", + "add rcx, 1408", + "mov r8d, 128", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 4096", + "mov rdx, rbx", + "add rdx, 34816", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 35840", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 6144", + "mov rdx, rbx", + "add rdx, 36864", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 7168", + "mov rdx, rbx", + "add rdx, 37888", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 8192", + "mov rdx, rbx", + "add rdx, 38912", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 39936", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 48128", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 47104", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 32", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 48128", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1536", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 9216", + "mov rdx, rbx", + "add rdx, 34816", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 10240", + "mov rdx, rbx", + "add rdx, 35840", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 11264", + "mov rdx, rbx", + "add rdx, 36864", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 12288", + "mov rdx, rbx", + "add rdx, 37888", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 38912", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 40960", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 48128", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 47104", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 352", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 48128", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1952", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 14336", + "mov rdx, rbx", + "add rdx, 34816", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 15360", + "mov rdx, rbx", + "add rdx, 35840", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 36864", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 17408", + "mov rdx, rbx", + "add rdx, 37888", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 18432", + "mov rdx, rbx", + "add rdx, 38912", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 41984", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 48128", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 47104", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 672", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 48128", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 2368", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 19456", + "mov rdx, rbx", + "add rdx, 34816", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 20480", + "mov rdx, rbx", + "add rdx, 35840", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 21504", + "mov rdx, rbx", + "add rdx, 36864", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 22528", + "mov rdx, rbx", + "add rdx, 37888", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 38912", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 43008", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 48128", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 47104", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 992", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 48128", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 2784", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 24576", + "mov rdx, rbx", + "add rdx, 34816", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 25600", + "mov rdx, rbx", + "add rdx, 35840", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 26624", + "mov rdx, rbx", + "add rdx, 36864", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 27648", + "mov rdx, rbx", + "add rdx, 37888", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 38912", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 44032", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 48128", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 47104", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 1312", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 48128", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 3200", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 34816", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 35840", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 36864", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 32768", + "mov rdx, rbx", + "add rdx, 37888", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 38912", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 45056", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 48128", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 47104", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 1632", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 48128", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 3616", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 1952", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 48", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r13", + "add rcx, 64", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, + vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, + vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + ) +} + +/// The CPU features `vg_mldsa65_sign_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA65_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-65 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address. +/// +/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing. +/// +/// # Safety +/// +/// * `sk` must be valid for reads of 4032 bytes. +/// * `mu` must be valid for reads of 64 bytes. +/// * `rnd` must be valid for reads of 32 bytes. +/// * `sig` must be valid for reads and writes of 3309 bytes. +/// * `scratch` must be valid for reads and writes of 103424 bytes. +/// * `sk` must have been written by `vg_mldsa65_keygen`. +/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do). +/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 3309], scratch: *mut [u64; 12928]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [r8+840], rbx", + "mov QWORD PTR [r8+848], rbp", + "mov QWORD PTR [r8+856], r12", + "mov QWORD PTR [r8+864], r13", + "mov QWORD PTR [r8+872], r14", + "mov QWORD PTR [r8+880], r15", + "mov rbx, r8", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r14, rcx", + "mov r15d, 1", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 51200", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 52224", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 57344", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 58368", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 59392", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 60416", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 61440", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 62464", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 63488", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 65536", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 67584", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 68608", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 69632", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 71680", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 72704", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 73728", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 74752", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 75776", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 76800", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 78848", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 79872", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "test r15d, r15d", + "jne 21f", + "jmp 22f", + "21:", + "mov rdi, rbp", + "add rdi, 128", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 32768", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 256", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 33792", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 384", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 34816", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 512", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 35840", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 640", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 36864", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 768", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 37888", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 896", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 38912", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1024", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 39936", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1152", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 40960", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 40960", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1280", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 41984", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 41984", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1408", + "mov esi, 128", + "mov edx, 4", + "mov ecx, 4", + "mov r8, rbx", + "add r8, 43008", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 43008", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1536", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 44032", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 44032", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1952", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 45056", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 45056", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 2368", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 46080", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 2784", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 47104", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 47104", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 3200", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 48128", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 48128", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 3616", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 49152", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 49152", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 32", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, r13", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 128", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 960", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov eax, 0", + "mov QWORD PTR [rbx+896], rax", + "mov eax, 814", + "mov QWORD PTR [rbx+888], rax", + "23:", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 0", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 16384", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 16384", + "mov ecx, 1024", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 24b", + "mov rdi, rbx", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 17408", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 17408", + "mov ecx, 1024", + "25:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 25b", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 18432", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 18432", + "mov ecx, 1024", + "26:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 26b", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 19456", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 19456", + "mov ecx, 1024", + "27:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 27b", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 4", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 20480", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 20480", + "mov ecx, 1024", + "28:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 28b", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 51200", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 52224", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 57344", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 58368", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 59392", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 60416", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 61440", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 62464", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 63488", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 65536", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 67584", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 68608", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 69632", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 71680", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 72704", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 73728", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 74752", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 75776", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 76800", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 23552", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 78848", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 79872", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2048", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2176", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 28672", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2304", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 29696", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2432", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 30720", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2560", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 31744", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2688", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, rbx", + "add rcx, 2048", + "mov r8d, 768", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 16", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1040", + "mov r8d, 48", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rdi, rbx", + "add rdi, 1040", + "mov esi, 48", + "mov edx, 49", + "mov rcx, rbx", + "add rcx, 5120", + "mov r8, rbx", + "add r8, 3072", + "call {vg_mldsa_sample_in_ball}", + "test eax, eax", + "jne 29f", + "mov r15d, 0", + "mov eax, 1", + "mov QWORD PTR [rbx+888], rax", + "jmp 210f", + "29:", + "mov rdi, rbx", + "add rdi, 5120", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov r15d, 1", + "mov eax, 0", + "mov QWORD PTR [rbx+904], rax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 32768", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 16384", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 16384", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 33792", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 17408", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 34816", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 18432", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 18432", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 35840", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 19456", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 36864", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 20480", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 20480", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 37888", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261692", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 38912", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261692", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 39936", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 28672", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261692", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 40960", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 29696", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261692", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 41984", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 30720", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261692", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 43008", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 31744", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261692", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 44032", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 26624", + "mov ecx, 1024", + "211:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 211b", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 26624", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 10240", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 45056", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 27648", + "mov ecx, 1024", + "212:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 212b", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 27648", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 27648", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 11264", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 46080", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 28672", + "mov ecx, 1024", + "213:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 213b", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 28672", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 28672", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 12288", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 47104", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 29696", + "mov ecx, 1024", + "214:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 214b", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 29696", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 29696", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 13312", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 48128", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 30720", + "mov ecx, 1024", + "215:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 215b", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 30720", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 30720", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 14336", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 49152", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 31744", + "mov ecx, 1024", + "216:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 216b", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 31744", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 31744", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 15360", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov eax, DWORD PTR [rbx+904]", + "sub rax, 56", + "shr rax, 63", + "and r15d, eax", + "test r15d, r15d", + "jne 217f", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 5", + "mov QWORD PTR [rbx+896], rax", + "jmp 218f", + "217:", + "mov eax, 1", + "mov QWORD PTR [rbx+888], rax", + "218:", + "210:", + "mov rax, QWORD PTR [rbx+888]", + "sub rax, 1", + "mov QWORD PTR [rbx+888], rax", + "jne 23b", + "test r15d, r15d", + "jne 219f", + "jmp 220f", + "219:", + "mov rdi, r14", + "add rdi, 0", + "mov rsi, rbx", + "add rsi, 1040", + "mov ecx, 48", + "221:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 221b", + "mov rdi, rbx", + "add rdi, 16384", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 48", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 17408", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 688", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 18432", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 1328", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 19456", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 1968", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 20480", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 2608", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 10240", + "mov esi, 1536", + "mov edx, 55", + "mov rcx, r14", + "add rcx, 3248", + "mov r8d, 61", + "call {vg_mldsa_hint_bit_pack}", + "220:", + "22:", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_high_bits = sym super::mldsa::vg_mldsa_high_bits, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, + vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, + vg_mldsa_low_bits = sym super::mldsa::vg_mldsa_low_bits, + vg_mldsa_make_hint = sym super::mldsa::vg_mldsa_make_hint, + vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, + vg_mldsa_hint_bit_pack = sym super::mldsa::vg_mldsa_hint_bit_pack, + ) +} + +/// The CPU features `vg_mldsa65_verify_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA65_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-65 verification of a signature of a message representative, `ML-DSA.Verify_internal(pk, M′, σ)` (FIPS 204 Algorithm 8) with `μ` computed by the caller: with the public key `*pk`, the 64-byte message representative `μ = H(H(pk, 64) ‖ M′, 64)` at `mu` and the signature `*sig`, returns 1 if the signature is valid, and 0 if it is not or if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less). +/// +/// Contract: `VG.Spec.MlDsa.verifyContract`. Not constant time: timing may depend on the public key, the message representative and the signature. +/// +/// It calls the `vg_mldsa_*` primitives and the SHAKE256 sponge. The samplers' results are combined without a branch, so the only branches depend on the public key and the signature. +/// +/// # Safety +/// +/// * `pk` must be valid for reads of 1952 bytes. +/// * `mu` must be valid for reads of 64 bytes. +/// * `sig` must be valid for reads of 3309 bytes. +/// * `scratch` must be valid for reads and writes of 103424 bytes. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952], mu: *const [u8; 64], sig: *const [u8; 3309], scratch: *mut [u64; 12928]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+840], rbx", + "mov QWORD PTR [rcx+848], rbp", + "mov QWORD PTR [rcx+856], r12", + "mov QWORD PTR [rcx+864], r13", + "mov QWORD PTR [rcx+872], r14", + "mov QWORD PTR [rcx+880], r15", + "mov rbx, rcx", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r15d, 1", + "mov rdi, r13", + "add rdi, 3248", + "mov esi, 61", + "mov edx, 55", + "mov rcx, rbx", + "add rcx, 8192", + "mov r8d, 1536", + "call {vg_mldsa_hint_bit_unpack}", + "mov r15d, eax", + "test r15d, r15d", + "jne 20f", + "jmp 21f", + "20:", + "mov rdi, r13", + "add rdi, 48", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 16384", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 16384", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 688", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 17408", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 17408", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 1328", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 18432", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 18432", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 1968", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 19456", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 19456", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 2608", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 20480", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 20480", + "mov esi, 524092", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "test r15d, r15d", + "jne 22f", + "jmp 23f", + "22:", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 24b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 28672", + "mov ecx, 256", + "25:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 25b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 29696", + "mov ecx, 256", + "26:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 26b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 30720", + "mov ecx, 256", + "27:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 27b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 31744", + "mov ecx, 256", + "28:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 28b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 32768", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 32768", + "mov ecx, 256", + "29:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 29b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 36864", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 36864", + "mov ecx, 256", + "210:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 210b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 37888", + "mov ecx, 256", + "211:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 211b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 38912", + "mov ecx, 256", + "212:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 212b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 39936", + "mov ecx, 256", + "213:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 213b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 40960", + "mov ecx, 256", + "214:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 45056", + "mov ecx, 256", + "215:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 215b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 46080", + "mov ecx, 256", + "216:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 216b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 47104", + "mov ecx, 256", + "217:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 217b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 48128", + "mov ecx, 256", + "218:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 218b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 49152", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 49152", + "mov ecx, 256", + "219:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 219b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 53248", + "mov ecx, 256", + "220:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 220b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 54272", + "mov ecx, 256", + "221:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 221b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 55296", + "mov ecx, 256", + "222:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 222b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 56320", + "mov ecx, 256", + "223:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 223b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 57344", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 57344", + "mov ecx, 256", + "224:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 224b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 61440", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 61440", + "mov ecx, 256", + "225:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 225b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 62464", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 62464", + "mov ecx, 256", + "226:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 226b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 63488", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 63488", + "mov ecx, 256", + "227:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 227b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 64512", + "mov ecx, 256", + "228:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 228b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 65536", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 65536", + "mov ecx, 256", + "229:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 229b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 69632", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 69632", + "mov ecx, 256", + "230:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 230b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 70656", + "mov ecx, 256", + "231:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 231b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 71680", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 71680", + "mov ecx, 256", + "232:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 232b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 72704", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 72704", + "mov ecx, 256", + "233:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 233b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 73728", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 73728", + "mov ecx, 256", + "234:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 234b", + "mov rdi, r13", + "add rdi, 0", + "mov esi, 48", + "mov edx, 49", + "mov rcx, rbx", + "add rcx, 23552", + "mov r8, rbx", + "add r8, 4096", + "call {vg_mldsa_sample_in_ball}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 23552", + "mov ecx, 256", + "235:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 235b", + "mov rdi, rbx", + "add rdi, 16384", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 18432", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 20480", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 32768", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1024", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 36864", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 352", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1152", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 49152", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 672", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 10240", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1280", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 57344", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 992", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 11264", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1408", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 61440", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 62464", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 63488", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 65536", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1312", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 12288", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1536", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 69632", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 71680", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 72704", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 73728", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1632", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 13312", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1664", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 768", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 16", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 2048", + "mov r8d, 48", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rsi, rbx", + "add rsi, 2048", + "mov rdi, r13", + "add rdi, 0", + "mov ecx, 48", + "mov edx, 0", + "236:", + "movzx eax, BYTE PTR [rsi]", + "movzx r8d, BYTE PTR [rdi]", + "xor rax, r8", + "or rdx, rax", + "add rsi, 1", + "add rdi, 1", + "sub rcx, 1", + "jne 236b", + "sub rdx, 1", + "sbb rax, rax", + "and r15d, eax", + "23:", + "21:", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, + vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, + vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1, + vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + ) +} + /// ML-DSA-65 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. /// /// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs index be3306d5c..79f684dd0 100644 --- a/src/asm/x86_64/mldsa87.rs +++ b/src/asm/x86_64/mldsa87.rs @@ -2,6 +2,7989 @@ //! Verified `mldsa87` functions for `x86_64`. #![allow(dead_code)] +/// The CPU features `vg_mldsa87_keygen_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA87_KEYGEN_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address. +/// +/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails. +/// +/// # Safety +/// +/// * `seed` must be valid for reads of 32 bytes. +/// * `pk` must be valid for reads and writes of 2592 bytes. +/// * `sk` must be valid for reads and writes of 4896 bytes. +/// * `scratch` must be valid for reads and writes of 144384 bytes. +/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do). +/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen_avx2(seed: *const [u8; 32], pk: *mut [u8; 2592], sk: *mut [u8; 4896], scratch: *mut [u64; 18048]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+840], rbx", + "mov QWORD PTR [rcx+848], rbp", + "mov QWORD PTR [rcx+856], r12", + "mov QWORD PTR [rcx+864], r13", + "mov QWORD PTR [rcx+872], r14", + "mov QWORD PTR [rcx+880], r15", + "mov rbx, rcx", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r15d, 1", + "mov eax, 8", + "mov BYTE PTR [rbx+896], al", + "mov eax, 7", + "mov BYTE PTR [rbx+897], al", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, rbx", + "add rcx, 896", + "mov r8d, 2", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 34", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 128", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", + "mov rdi, rbx", + "add rdi, 1216", + "mov rsi, rbx", + "add rsi, 1056", + "mov ecx, 64", + "21:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 21b", + "mov eax, 0", + "mov BYTE PTR [rbx+1281], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 4096", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 4096", + "mov ecx, 256", + "22:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 22b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 5120", + "mov ecx, 256", + "23:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 23b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 6144", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov ecx, 256", + "24:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 24b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 7168", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 7168", + "mov ecx, 256", + "25:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 25b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 8192", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 8192", + "mov ecx, 256", + "26:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 26b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 9216", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov ecx, 256", + "27:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 27b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 10240", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 10240", + "mov ecx, 256", + "28:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 28b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 11264", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 11264", + "mov ecx, 256", + "29:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 29b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 12288", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 12288", + "mov ecx, 256", + "210:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 210b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 13312", + "mov ecx, 256", + "211:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 211b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 14336", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 14336", + "mov ecx, 256", + "212:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 212b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 15360", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 15360", + "mov ecx, 256", + "213:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 213b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 16384", + "mov ecx, 256", + "214:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 214b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 17408", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 17408", + "mov ecx, 256", + "215:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 215b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 18432", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 18432", + "mov ecx, 256", + "216:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 216b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 19456", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 19456", + "mov ecx, 256", + "217:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 217b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 20480", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 20480", + "mov ecx, 256", + "218:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 218b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 21504", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 21504", + "mov ecx, 256", + "219:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 219b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 22528", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 22528", + "mov ecx, 256", + "220:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 220b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 23552", + "mov ecx, 256", + "221:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 221b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 24576", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 24576", + "mov ecx, 256", + "222:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 222b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 25600", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 25600", + "mov ecx, 256", + "223:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 223b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 26624", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 26624", + "mov ecx, 256", + "224:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 224b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 27648", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 27648", + "mov ecx, 256", + "225:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 225b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 28672", + "mov ecx, 256", + "226:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 226b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 29696", + "mov ecx, 256", + "227:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 227b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 30720", + "mov ecx, 256", + "228:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 228b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 31744", + "mov ecx, 256", + "229:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 229b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 32768", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 32768", + "mov ecx, 256", + "230:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 230b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 33792", + "mov ecx, 256", + "231:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 231b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 34816", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 34816", + "mov ecx, 256", + "232:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 232b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 35840", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 35840", + "mov ecx, 256", + "233:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 233b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 36864", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 36864", + "mov ecx, 256", + "234:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 234b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 37888", + "mov ecx, 256", + "235:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 235b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 38912", + "mov ecx, 256", + "236:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 236b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 39936", + "mov ecx, 256", + "237:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 237b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 40960", + "mov ecx, 256", + "238:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 238b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 41984", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 41984", + "mov ecx, 256", + "239:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 239b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 43008", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 43008", + "mov ecx, 256", + "240:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 240b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 44032", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 44032", + "mov ecx, 256", + "241:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 241b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 45056", + "mov ecx, 256", + "242:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 242b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 46080", + "mov ecx, 256", + "243:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 243b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 47104", + "mov ecx, 256", + "244:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 244b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 48128", + "mov ecx, 256", + "245:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 245b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 49152", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 49152", + "mov ecx, 256", + "246:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 246b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 50176", + "mov ecx, 256", + "247:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 247b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 51200", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 51200", + "mov ecx, 256", + "248:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 248b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 52224", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 52224", + "mov ecx, 256", + "249:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 249b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 53248", + "mov ecx, 256", + "250:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 250b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 54272", + "mov ecx, 256", + "251:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 251b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 55296", + "mov ecx, 256", + "252:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 252b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 56320", + "mov ecx, 256", + "253:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 253b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 57344", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 57344", + "mov ecx, 256", + "254:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 254b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 58368", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 58368", + "mov ecx, 256", + "255:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 255b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 59392", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 59392", + "mov ecx, 256", + "256:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 256b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 60416", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 60416", + "mov ecx, 256", + "257:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 257b", + "mov eax, 0", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 61440", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 61440", + "mov ecx, 256", + "258:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 258b", + "mov eax, 1", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 62464", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 62464", + "mov ecx, 256", + "259:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 259b", + "mov eax, 2", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 63488", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 63488", + "mov ecx, 256", + "260:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 260b", + "mov eax, 3", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 64512", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 64512", + "mov ecx, 256", + "261:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 261b", + "mov eax, 4", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 65536", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 65536", + "mov ecx, 256", + "262:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 262b", + "mov eax, 5", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 66560", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 66560", + "mov ecx, 256", + "263:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 263b", + "mov eax, 6", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 67584", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 67584", + "mov ecx, 256", + "264:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 264b", + "mov eax, 7", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 68608", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 68608", + "mov ecx, 256", + "265:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 265b", + "mov eax, 8", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 69632", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 69632", + "mov ecx, 256", + "266:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 266b", + "mov eax, 9", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 70656", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 70656", + "mov ecx, 256", + "267:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 267b", + "mov eax, 10", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 71680", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 71680", + "mov ecx, 256", + "268:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 268b", + "mov eax, 11", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 72704", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 72704", + "mov ecx, 256", + "269:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 269b", + "mov eax, 12", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 73728", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 73728", + "mov ecx, 256", + "270:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 270b", + "mov eax, 13", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 74752", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 74752", + "mov ecx, 256", + "271:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 271b", + "mov eax, 14", + "mov BYTE PTR [rbx+1280], al", + "mov rdi, rbx", + "add rdi, 1216", + "mov esi, 2", + "mov rdx, rbx", + "add rdx, 75776", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 75776", + "mov ecx, 256", + "272:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 272b", + "mov rdi, r12", + "add rdi, 0", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "273:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 273b", + "mov rdi, r13", + "add rdi, 0", + "mov rsi, rbx", + "add rsi, 1024", + "mov ecx, 32", + "274:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 274b", + "mov rdi, r13", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 1120", + "mov ecx, 32", + "275:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 275b", + "mov rdi, rbx", + "add rdi, 61440", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 128", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 62464", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 224", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 63488", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 320", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 64512", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 416", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 65536", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 512", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 66560", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 608", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 67584", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 704", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 68608", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 800", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 69632", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 896", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 70656", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 992", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 71680", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1088", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 72704", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1184", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 73728", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1280", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 74752", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1376", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 75776", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1472", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 61440", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 62464", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 63488", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 64512", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 65536", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 66560", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 67584", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 4096", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 6144", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 7168", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 8192", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 9216", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 10240", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 68608", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 32", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1568", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 11264", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 12288", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 14336", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 15360", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 17408", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 69632", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 352", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1984", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 18432", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 19456", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 20480", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 21504", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 22528", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 24576", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 70656", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 672", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 2400", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 25600", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 26624", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 27648", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 71680", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 992", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 2816", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 32768", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 34816", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 35840", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 36864", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 72704", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 1312", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 3232", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 41984", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 43008", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 44032", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 73728", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 1632", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 3648", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 49152", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 51200", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 52224", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 74752", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 1952", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 4064", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 57344", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 58368", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 59392", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 60416", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 75776", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", + "mov rdi, rbx", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 2272", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 4480", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 2592", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 8", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r13", + "add rcx, 64", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_bounded_poly = sym super::mldsa::vg_mldsa_rej_bounded_poly, + vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, + vg_mldsa_power2round = sym super::mldsa::vg_mldsa_power2round, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + ) +} + +/// The CPU features `vg_mldsa87_sign_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA87_SIGN_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-87 signing of a message representative, `ML-DSA.Sign_internal(sk, M′, rnd)` (FIPS 204 Algorithm 7) with `μ` computed by the caller: with the private key `*sk`, the 64-byte message representative `μ = H(tr ‖ M′, 64)` at `mu` (for the public key hash `tr`, bytes 64–127 of `*sk`) and the randomness `*rnd`, writes the signature to `*sig`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.signContract`. Constant time but for `ρ` and the rejection sampling: timing may depend on the pointers, on `ρ` (the first 32 bytes of `*sk`), on the number of iterations of the signing loop and the commitment hash of each, and on the hint of the signature (`signLeak`), but not on anything else of the key, the message representative or the randomness. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 24 bytes of stack below its return address. +/// +/// The signing loop runs at most 814 iterations (FIPS 204 Appendix C). Each iteration computes every validity check and combines them without branching: the one branch on their result is the only place an iteration's outcome affects timing. +/// +/// # Safety +/// +/// * `sk` must be valid for reads of 4896 bytes. +/// * `mu` must be valid for reads of 64 bytes. +/// * `rnd` must be valid for reads of 32 bytes. +/// * `sig` must be valid for reads and writes of 4627 bytes. +/// * `scratch` must be valid for reads and writes of 144384 bytes. +/// * `sk` must have been written by `vg_mldsa87_keygen`. +/// * `rnd` must be fresh random bytes (FIPS 204 §3.6.1), or 32 zero bytes for deterministic signing. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `sig` and `scratch` must not overlap each other, `sk`, `mu` or `rnd` (distinct Rust objects never do). +/// * None of `sk`, `mu`, `rnd`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], mu: *const [u8; 64], rnd: *const [u8; 32], sig: *mut [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [r8+840], rbx", + "mov QWORD PTR [r8+848], rbp", + "mov QWORD PTR [r8+856], r12", + "mov QWORD PTR [r8+864], r13", + "mov QWORD PTR [r8+872], r14", + "mov QWORD PTR [r8+880], r15", + "mov rbx, r8", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r14, rcx", + "mov r15d, 1", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 65536", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 67584", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 68608", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 69632", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 0", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 71680", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 72704", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 73728", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 74752", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 75776", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 76800", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 1", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 78848", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 79872", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 80896", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 81920", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 82944", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 83968", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 2", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 84992", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 86016", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 87040", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 88064", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 89088", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 90112", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 91136", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 3", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 92160", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 93184", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 94208", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 95232", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 96256", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 97280", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 98304", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 4", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 99328", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 100352", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 101376", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 102400", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 103424", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 104448", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 105472", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 5", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 106496", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 107520", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 108544", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 109568", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 110592", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 111616", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 112640", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 6", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 113664", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 0", + "mov BYTE PTR [rbx+944], al", + "mov eax, 7", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 114688", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 1", + "mov BYTE PTR [rbx+944], al", + "mov eax, 7", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 115712", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 2", + "mov BYTE PTR [rbx+944], al", + "mov eax, 7", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 116736", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 3", + "mov BYTE PTR [rbx+944], al", + "mov eax, 7", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 117760", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 4", + "mov BYTE PTR [rbx+944], al", + "mov eax, 7", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 118784", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 5", + "mov BYTE PTR [rbx+944], al", + "mov eax, 7", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 119808", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov eax, 6", + "mov BYTE PTR [rbx+944], al", + "mov eax, 7", + "mov BYTE PTR [rbx+945], al", + "mov rdi, rbx", + "add rdi, 912", + "mov rsi, rbx", + "add rsi, 120832", + "mov rdx, rbx", + "add rdx, 3072", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "test r15d, r15d", + "jne 21f", + "jmp 22f", + "21:", + "mov rdi, rbp", + "add rdi, 128", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 40960", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 40960", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 224", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 41984", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 41984", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 320", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 43008", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 43008", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 416", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 44032", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 44032", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 512", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 45056", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 45056", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 608", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 46080", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 46080", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 704", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 47104", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 47104", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 800", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 48128", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 48128", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 896", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 49152", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 49152", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 992", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 50176", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 50176", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1088", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 51200", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 51200", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1184", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 52224", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 52224", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1280", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 53248", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 53248", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1376", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 54272", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 54272", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1472", + "mov esi, 96", + "mov edx, 2", + "mov ecx, 2", + "mov r8, rbx", + "add r8, 55296", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 55296", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1568", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 56320", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 56320", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1984", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 57344", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 57344", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 2400", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 58368", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 58368", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 2816", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 59392", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 59392", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 3232", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 60416", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 60416", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 3648", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 61440", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 61440", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 4064", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 62464", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 62464", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 4480", + "mov esi, 416", + "mov edx, 4095", + "mov ecx, 4096", + "mov r8, rbx", + "add r8, 63488", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 63488", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 32", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, r13", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 128", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 960", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov eax, 0", + "mov QWORD PTR [rbx+896], rax", + "mov eax, 814", + "mov QWORD PTR [rbx+888], rax", + "23:", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 0", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 18432", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 18432", + "mov ecx, 1024", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 24b", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 19456", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 19456", + "mov ecx, 1024", + "25:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 25b", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 20480", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 20480", + "mov ecx, 1024", + "26:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 26b", + "mov rdi, rbx", + "add rdi, 27648", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 21504", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 21504", + "mov ecx, 1024", + "27:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 27b", + "mov rdi, rbx", + "add rdi, 28672", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 4", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 22528", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 22528", + "mov ecx, 1024", + "28:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 28b", + "mov rdi, rbx", + "add rdi, 29696", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 5", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 23552", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 23552", + "mov ecx, 1024", + "29:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 29b", + "mov rdi, rbx", + "add rdi, 30720", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 6", + "mov BYTE PTR [rbx+1024], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1025], al", + "mov rdi, rbx", + "add rdi, 960", + "mov esi, 524288", + "mov rdx, rbx", + "add rdx, 24576", + "mov rcx, rbx", + "add rcx, 3072", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 24576", + "mov ecx, 1024", + "210:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 210b", + "mov rdi, rbx", + "add rdi, 31744", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 65536", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 67584", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 68608", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 69632", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 71680", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 72704", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 73728", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 74752", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 75776", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 76800", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 78848", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 79872", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 80896", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 81920", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 82944", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 83968", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 84992", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 86016", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 87040", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 88064", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 89088", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 90112", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 91136", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 92160", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 93184", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 94208", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 95232", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 96256", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 97280", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 98304", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 99328", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 100352", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 101376", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 102400", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 103424", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 104448", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 105472", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 106496", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 107520", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 108544", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 109568", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 110592", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 111616", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 112640", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 113664", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 114688", + "mov rdx, rbx", + "add rdx, 25600", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 115712", + "mov rdx, rbx", + "add rdx, 26624", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 116736", + "mov rdx, rbx", + "add rdx, 27648", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 117760", + "mov rdx, rbx", + "add rdx, 28672", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 118784", + "mov rdx, rbx", + "add rdx, 29696", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 119808", + "mov rdx, rbx", + "add rdx, 30720", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 120832", + "mov rdx, rbx", + "add rdx, 31744", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2048", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 33792", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2176", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 34816", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2304", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 35840", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2432", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 36864", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2560", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 37888", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2688", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 38912", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2816", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 39936", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_high_bits}", + "mov rdi, rbx", + "add rdi, 6144", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 2944", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, rbx", + "add rcx, 2048", + "mov r8d, 1024", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1040", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rdi, rbx", + "add rdi, 1040", + "mov esi, 64", + "mov edx, 60", + "mov rcx, rbx", + "add rcx, 5120", + "mov r8, rbx", + "add r8, 3072", + "call {vg_mldsa_sample_in_ball}", + "test eax, eax", + "jne 211f", + "mov r15d, 0", + "mov eax, 1", + "mov QWORD PTR [rbx+888], rax", + "jmp 212f", + "211:", + "mov rdi, rbx", + "add rdi, 5120", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_ntt_avx2}", + "mov r15d, 1", + "mov eax, 0", + "mov QWORD PTR [rbx+904], rax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 40960", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 18432", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 18432", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 41984", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 19456", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 43008", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 20480", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 20480", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 44032", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 21504", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 45056", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 46080", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 47104", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 24576", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 48128", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 32768", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261768", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 49152", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 33792", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261768", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 50176", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 34816", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261768", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 51200", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 35840", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261768", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 52224", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 36864", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261768", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 53248", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 37888", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261768", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 54272", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 38912", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261768", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 55296", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 6144", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 6144", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 39936", + "mov esi, 261888", + "mov rdx, rbx", + "add rdx, 7168", + "call {vg_mldsa_low_bits}", + "mov rdi, rbx", + "add rdi, 7168", + "mov esi, 261768", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 56320", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 32768", + "mov ecx, 1024", + "213:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 213b", + "mov rdi, rbx", + "add rdi, 32768", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 32768", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 32768", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 10240", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 57344", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 33792", + "mov ecx, 1024", + "214:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 214b", + "mov rdi, rbx", + "add rdi, 33792", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 33792", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 33792", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 11264", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 58368", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 34816", + "mov ecx, 1024", + "215:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 215b", + "mov rdi, rbx", + "add rdi, 34816", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 34816", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 34816", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 12288", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 59392", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 35840", + "mov ecx, 1024", + "216:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 216b", + "mov rdi, rbx", + "add rdi, 35840", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 35840", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 35840", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 13312", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 60416", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 36864", + "mov ecx, 1024", + "217:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 217b", + "mov rdi, rbx", + "add rdi, 36864", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 36864", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 36864", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 14336", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 37888", + "mov ecx, 1024", + "218:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 218b", + "mov rdi, rbx", + "add rdi, 37888", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 37888", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 37888", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 15360", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 38912", + "mov ecx, 1024", + "219:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 219b", + "mov rdi, rbx", + "add rdi, 38912", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 38912", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 38912", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 16384", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 3072", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov esi, 261888", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 39936", + "mov ecx, 1024", + "220:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 220b", + "mov rdi, rbx", + "add rdi, 39936", + "mov rsi, rbx", + "add rsi, 8192", + "call {vg_mldsa_add_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 39936", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 39936", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 17408", + "call {vg_mldsa_make_hint}", + "mov ecx, DWORD PTR [rbx+904]", + "add ecx, eax", + "mov QWORD PTR [rbx+904], rcx", + "mov eax, DWORD PTR [rbx+904]", + "sub rax, 76", + "shr rax, 63", + "and r15d, eax", + "test r15d, r15d", + "jne 221f", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 7", + "mov QWORD PTR [rbx+896], rax", + "jmp 222f", + "221:", + "mov eax, 1", + "mov QWORD PTR [rbx+888], rax", + "222:", + "212:", + "mov rax, QWORD PTR [rbx+888]", + "sub rax, 1", + "mov QWORD PTR [rbx+888], rax", + "jne 23b", + "test r15d, r15d", + "jne 223f", + "jmp 224f", + "223:", + "mov rdi, r14", + "add rdi, 0", + "mov rsi, rbx", + "add rsi, 1040", + "mov ecx, 64", + "225:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 225b", + "mov rdi, rbx", + "add rdi, 18432", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 64", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 19456", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 704", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 20480", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 1344", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 21504", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 1984", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 22528", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 2624", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 23552", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 3264", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 24576", + "mov esi, 524287", + "mov edx, 524288", + "mov rcx, r14", + "add rcx, 3904", + "mov r8d, 640", + "call {vg_mldsa_bit_pack}", + "mov rdi, rbx", + "add rdi, 10240", + "mov esi, 2048", + "mov edx, 75", + "mov rcx, r14", + "add rcx, 4544", + "mov r8d, 83", + "call {vg_mldsa_hint_bit_pack}", + "224:", + "22:", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_high_bits = sym super::mldsa::vg_mldsa_high_bits, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, + vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, + vg_mldsa_low_bits = sym super::mldsa::vg_mldsa_low_bits, + vg_mldsa_make_hint = sym super::mldsa::vg_mldsa_make_hint, + vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, + vg_mldsa_hint_bit_pack = sym super::mldsa::vg_mldsa_hint_bit_pack, + ) +} + +/// The CPU features `vg_mldsa87_verify_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA87_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// ML-DSA-87 verification of a signature of a message representative, `ML-DSA.Verify_internal(pk, M′, σ)` (FIPS 204 Algorithm 8) with `μ` computed by the caller: with the public key `*pk`, the 64-byte message representative `μ = H(H(pk, 64) ‖ M′, 64)` at `mu` and the signature `*sig`, returns 1 if the signature is valid, and 0 if it is not or if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less). +/// +/// Contract: `VG.Spec.MlDsa.verifyContract`. Not constant time: timing may depend on the public key, the message representative and the signature. +/// +/// It calls the `vg_mldsa_*` primitives and the SHAKE256 sponge. The samplers' results are combined without a branch, so the only branches depend on the public key and the signature. +/// +/// # Safety +/// +/// * `pk` must be valid for reads of 2592 bytes. +/// * `mu` must be valid for reads of 64 bytes. +/// * `sig` must be valid for reads of 4627 bytes. +/// * `scratch` must be valid for reads and writes of 144384 bytes. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592], mu: *const [u8; 64], sig: *const [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+840], rbx", + "mov QWORD PTR [rcx+848], rbp", + "mov QWORD PTR [rcx+856], r12", + "mov QWORD PTR [rcx+864], r13", + "mov QWORD PTR [rcx+872], r14", + "mov QWORD PTR [rcx+880], r15", + "mov rbx, rcx", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r15d, 1", + "mov rdi, r13", + "add rdi, 4544", + "mov esi, 83", + "mov edx, 75", + "mov rcx, rbx", + "add rcx, 8192", + "mov r8d, 2048", + "call {vg_mldsa_hint_bit_unpack}", + "mov r15d, eax", + "test r15d, r15d", + "jne 20f", + "jmp 21f", + "20:", + "mov rdi, r13", + "add rdi, 64", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 16384", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 16384", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 704", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 17408", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 17408", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 1344", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 18432", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 18432", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 1984", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 19456", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 19456", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 2624", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 20480", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 20480", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 3264", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 21504", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 21504", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 3904", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 22528", + "call {vg_mldsa_bit_unpack}", + "mov rdi, rbx", + "add rdi, 22528", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "test r15d, r15d", + "jne 22f", + "jmp 23f", + "22:", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 24b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 28672", + "mov ecx, 256", + "25:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 25b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 29696", + "mov ecx, 256", + "26:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 26b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 30720", + "mov ecx, 256", + "27:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 27b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 31744", + "mov ecx, 256", + "28:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 28b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 32768", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 32768", + "mov ecx, 256", + "29:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 29b", + "mov eax, 5", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 33792", + "mov ecx, 256", + "210:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 210b", + "mov eax, 6", + "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 34816", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 34816", + "mov ecx, 256", + "211:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 211b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 36864", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 36864", + "mov ecx, 256", + "212:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 212b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 37888", + "mov ecx, 256", + "213:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 213b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 38912", + "mov ecx, 256", + "214:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 214b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 39936", + "mov ecx, 256", + "215:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 215b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 40960", + "mov ecx, 256", + "216:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 216b", + "mov eax, 5", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 41984", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 41984", + "mov ecx, 256", + "217:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 217b", + "mov eax, 6", + "mov BYTE PTR [rbx+928], al", + "mov eax, 1", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 43008", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 43008", + "mov ecx, 256", + "218:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 218b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 45056", + "mov ecx, 256", + "219:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 219b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 46080", + "mov ecx, 256", + "220:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 220b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 47104", + "mov ecx, 256", + "221:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 221b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 48128", + "mov ecx, 256", + "222:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 222b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 49152", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 49152", + "mov ecx, 256", + "223:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 223b", + "mov eax, 5", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 50176", + "mov ecx, 256", + "224:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 224b", + "mov eax, 6", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 51200", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 51200", + "mov ecx, 256", + "225:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 225b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 53248", + "mov ecx, 256", + "226:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 226b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 54272", + "mov ecx, 256", + "227:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 227b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 55296", + "mov ecx, 256", + "228:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 228b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 56320", + "mov ecx, 256", + "229:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 229b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 57344", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 57344", + "mov ecx, 256", + "230:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 230b", + "mov eax, 5", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 58368", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 58368", + "mov ecx, 256", + "231:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 231b", + "mov eax, 6", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 59392", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 59392", + "mov ecx, 256", + "232:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 232b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 61440", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 61440", + "mov ecx, 256", + "233:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 233b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 62464", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 62464", + "mov ecx, 256", + "234:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 234b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 63488", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 63488", + "mov ecx, 256", + "235:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 235b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 64512", + "mov ecx, 256", + "236:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 236b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 65536", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 65536", + "mov ecx, 256", + "237:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 237b", + "mov eax, 5", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 66560", + "mov ecx, 256", + "238:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 238b", + "mov eax, 6", + "mov BYTE PTR [rbx+928], al", + "mov eax, 4", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 67584", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 67584", + "mov ecx, 256", + "239:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 239b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 69632", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 69632", + "mov ecx, 256", + "240:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 240b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 70656", + "mov ecx, 256", + "241:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 241b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 71680", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 71680", + "mov ecx, 256", + "242:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 242b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 72704", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 72704", + "mov ecx, 256", + "243:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 243b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 73728", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 73728", + "mov ecx, 256", + "244:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 244b", + "mov eax, 5", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 74752", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 74752", + "mov ecx, 256", + "245:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 245b", + "mov eax, 6", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 75776", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 75776", + "mov ecx, 256", + "246:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 246b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 6", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 77824", + "mov ecx, 256", + "247:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 247b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 6", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 78848", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 78848", + "mov ecx, 256", + "248:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 248b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 6", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 79872", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 79872", + "mov ecx, 256", + "249:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 249b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 6", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 80896", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 80896", + "mov ecx, 256", + "250:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 250b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 6", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 81920", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 81920", + "mov ecx, 256", + "251:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 251b", + "mov eax, 5", + "mov BYTE PTR [rbx+928], al", + "mov eax, 6", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 82944", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 82944", + "mov ecx, 256", + "252:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 252b", + "mov eax, 6", + "mov BYTE PTR [rbx+928], al", + "mov eax, 6", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 83968", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 83968", + "mov ecx, 256", + "253:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 253b", + "mov eax, 0", + "mov BYTE PTR [rbx+928], al", + "mov eax, 7", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 86016", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 86016", + "mov ecx, 256", + "254:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 254b", + "mov eax, 1", + "mov BYTE PTR [rbx+928], al", + "mov eax, 7", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 87040", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 87040", + "mov ecx, 256", + "255:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 255b", + "mov eax, 2", + "mov BYTE PTR [rbx+928], al", + "mov eax, 7", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 88064", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 88064", + "mov ecx, 256", + "256:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 256b", + "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 7", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 89088", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 89088", + "mov ecx, 256", + "257:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 257b", + "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 7", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 90112", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 90112", + "mov ecx, 256", + "258:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 258b", + "mov eax, 5", + "mov BYTE PTR [rbx+928], al", + "mov eax, 7", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 91136", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 91136", + "mov ecx, 256", + "259:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 259b", + "mov eax, 6", + "mov BYTE PTR [rbx+928], al", + "mov eax, 7", + "mov BYTE PTR [rbx+929], al", + "mov rdi, rbx", + "add rdi, 896", + "mov rsi, rbx", + "add rsi, 92160", + "mov rdx, rbx", + "add rdx, 4096", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 92160", + "mov ecx, 256", + "260:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 260b", + "mov rdi, r13", + "add rdi, 0", + "mov esi, 64", + "mov edx, 60", + "mov rcx, rbx", + "add rcx, 23552", + "mov r8, rbx", + "add r8, 4096", + "call {vg_mldsa_sample_in_ball}", + "and r15d, eax", + "mov edx, 0", + "sub edx, eax", + "mov rdi, rbx", + "add rdi, 23552", + "mov ecx, 256", + "261:", + "mov eax, DWORD PTR [rdi]", + "and eax, edx", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 261b", + "mov rdi, rbx", + "add rdi, 16384", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 18432", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 20480", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 22528", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 30720", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 32768", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 34816", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1024", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 36864", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 38912", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 40960", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 41984", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 43008", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 352", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1152", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 45056", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 47104", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 49152", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 51200", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 672", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 10240", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1280", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 53248", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 55296", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 56320", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 57344", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 58368", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 59392", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 992", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 11264", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1408", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 61440", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 62464", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 63488", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 65536", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 67584", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1312", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 12288", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1536", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 69632", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 70656", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 71680", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 72704", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 73728", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 74752", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 75776", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1632", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 13312", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1664", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 78848", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 79872", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 80896", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 81920", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 82944", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 83968", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1952", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 14336", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1792", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 86016", + "mov rdx, rbx", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 87040", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 88064", + "mov rdx, rbx", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 89088", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 90112", + "mov rdx, rbx", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 91136", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 92160", + "mov rdx, rbx", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 2272", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", + "mov rdi, rbx", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 25600", + "mov rsi, rbx", + "add rsi, 23552", + "mov rdx, rbx", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", + "mov rdi, rbx", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", + "mov rdi, rbx", + "add rdi, 15360", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", + "mov rdi, rbx", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1920", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 1024", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", + "mov rdi, rbx", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 2048", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rsi, rbx", + "add rsi, 2048", + "mov rdi, r13", + "add rdi, 0", + "mov ecx, 64", + "mov edx, 0", + "262:", + "movzx eax, BYTE PTR [rsi]", + "movzx r8d, BYTE PTR [rdi]", + "xor rax, r8", + "or rdx, rax", + "add rsi, 1", + "add rdi, 1", + "sub rcx, 1", + "jne 262b", + "sub rdx, 1", + "sbb rax, rax", + "and r15d, eax", + "23:", + "21:", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, + vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, + vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1, + vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + ) +} + /// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. /// /// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. diff --git a/src/mldsa44.rs b/src/mldsa44.rs index 782c59c4b..04777dad9 100644 --- a/src/mldsa44.rs +++ b/src/mldsa44.rs @@ -38,6 +38,9 @@ crate::mldsa_common::ml_dsa! { keygen_sha3: (crate::arch::mldsa44::vg_mldsa44_keygen_sha3, crate::arch::mldsa44::VG_MLDSA44_KEYGEN_SHA3_FEATURES), sign_sha3: (crate::arch::mldsa44::vg_mldsa44_sign_sha3, crate::arch::mldsa44::VG_MLDSA44_SIGN_SHA3_FEATURES), verify_sha3: (crate::arch::mldsa44::vg_mldsa44_verify_sha3, crate::arch::mldsa44::VG_MLDSA44_VERIFY_SHA3_FEATURES), + keygen_avx2: (crate::arch::mldsa44::vg_mldsa44_keygen_avx2, crate::arch::mldsa44::VG_MLDSA44_KEYGEN_AVX2_FEATURES), + sign_avx2: (crate::arch::mldsa44::vg_mldsa44_sign_avx2, crate::arch::mldsa44::VG_MLDSA44_SIGN_AVX2_FEATURES), + verify_avx2: (crate::arch::mldsa44::vg_mldsa44_verify_avx2, crate::arch::mldsa44::VG_MLDSA44_VERIFY_AVX2_FEATURES), pk: 1312, sk: 2560, sig: 2420, diff --git a/src/mldsa65.rs b/src/mldsa65.rs index a67072013..79dff1485 100644 --- a/src/mldsa65.rs +++ b/src/mldsa65.rs @@ -38,6 +38,9 @@ crate::mldsa_common::ml_dsa! { keygen_sha3: (crate::arch::mldsa65::vg_mldsa65_keygen_sha3, crate::arch::mldsa65::VG_MLDSA65_KEYGEN_SHA3_FEATURES), sign_sha3: (crate::arch::mldsa65::vg_mldsa65_sign_sha3, crate::arch::mldsa65::VG_MLDSA65_SIGN_SHA3_FEATURES), verify_sha3: (crate::arch::mldsa65::vg_mldsa65_verify_sha3, crate::arch::mldsa65::VG_MLDSA65_VERIFY_SHA3_FEATURES), + keygen_avx2: (crate::arch::mldsa65::vg_mldsa65_keygen_avx2, crate::arch::mldsa65::VG_MLDSA65_KEYGEN_AVX2_FEATURES), + sign_avx2: (crate::arch::mldsa65::vg_mldsa65_sign_avx2, crate::arch::mldsa65::VG_MLDSA65_SIGN_AVX2_FEATURES), + verify_avx2: (crate::arch::mldsa65::vg_mldsa65_verify_avx2, crate::arch::mldsa65::VG_MLDSA65_VERIFY_AVX2_FEATURES), pk: 1952, sk: 4032, sig: 3309, diff --git a/src/mldsa87.rs b/src/mldsa87.rs index 7b672b3e4..ce1e72136 100644 --- a/src/mldsa87.rs +++ b/src/mldsa87.rs @@ -38,6 +38,9 @@ crate::mldsa_common::ml_dsa! { keygen_sha3: (crate::arch::mldsa87::vg_mldsa87_keygen_sha3, crate::arch::mldsa87::VG_MLDSA87_KEYGEN_SHA3_FEATURES), sign_sha3: (crate::arch::mldsa87::vg_mldsa87_sign_sha3, crate::arch::mldsa87::VG_MLDSA87_SIGN_SHA3_FEATURES), verify_sha3: (crate::arch::mldsa87::vg_mldsa87_verify_sha3, crate::arch::mldsa87::VG_MLDSA87_VERIFY_SHA3_FEATURES), + keygen_avx2: (crate::arch::mldsa87::vg_mldsa87_keygen_avx2, crate::arch::mldsa87::VG_MLDSA87_KEYGEN_AVX2_FEATURES), + sign_avx2: (crate::arch::mldsa87::vg_mldsa87_sign_avx2, crate::arch::mldsa87::VG_MLDSA87_SIGN_AVX2_FEATURES), + verify_avx2: (crate::arch::mldsa87::vg_mldsa87_verify_avx2, crate::arch::mldsa87::VG_MLDSA87_VERIFY_AVX2_FEATURES), pk: 2592, sk: 4896, sig: 4627, diff --git a/src/mldsa_common.rs b/src/mldsa_common.rs index 2979826b0..448792854 100644 --- a/src/mldsa_common.rs +++ b/src/mldsa_common.rs @@ -25,6 +25,45 @@ pub(crate) fn message_rep(tr: &[u8; 64], msg: &[u8], ctx: &[u8]) -> Option<[u8; Some(mu) } +/// The implementations of ML-DSA's verified functions: their instances for +/// the polynomial arithmetic and Keccak they call (`Generic/MlDsaArith/` and +/// the Keccak backend, `crate::hashes::sha3::Backend`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Backend { + /// The target's baseline ISA (on x86-64, SSE2 polynomial arithmetic), + /// with scalar Keccak. + Scalar, + /// AArch64 with the SHA-3 extension, for Keccak. + #[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))] + Sha3, + /// x86-64 with AVX2, for the polynomial arithmetic. + #[cfg(target_arch = "x86_64")] + Avx2, +} + +impl Backend { + /// The implementation for the Keccak backend `keccak`, on a CPU with the + /// features `f`, of functions whose AVX2 instances need `avx2`. + #[cfg_attr(not(target_arch = "x86_64"), allow(unused_variables))] + pub(crate) fn select( + keccak: crate::hashes::sha3::Backend, + f: crate::cpu::Features, + avx2: &[&[&str]], + ) -> Backend { + match keccak { + crate::hashes::sha3::Backend::Scalar => { + #[cfg(target_arch = "x86_64")] + if f.contains(crate::cpu::Features::all(avx2)) { + return Backend::Avx2; + } + Backend::Scalar + } + #[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))] + crate::hashes::sha3::Backend::Sha3 => Backend::Sha3, + } + } +} + /// Defines the API of one ML-DSA parameter set: its `Error`, `SigningKey` /// and `VerifyingKey`, over its verified `keygen`, `sign` and `verify` /// functions (which take `μ`) and its sizes. @@ -39,6 +78,9 @@ macro_rules! ml_dsa { keygen_sha3: ($keygen_sha3:path, $keygen_sha3_features:path), sign_sha3: ($sign_sha3:path, $sign_sha3_features:path), verify_sha3: ($verify_sha3:path, $verify_sha3_features:path), + keygen_avx2: ($keygen_avx2:path, $keygen_avx2_features:path), + sign_avx2: ($sign_avx2:path, $sign_avx2_features:path), + verify_avx2: ($verify_avx2:path, $verify_avx2_features:path), pk: $pk:literal, sk: $sk:literal, sig: $sig:literal, @@ -46,10 +88,12 @@ macro_rules! ml_dsa { ) => { use $crate::mldsa_common::message_rep; use $crate::zeroize::zeroize; - use $crate::hashes::sha3::Backend; + use $crate::mldsa_common::Backend; - /// Follow the shared Keccak backend only when all generated callers' - /// feature requirements are met. The draft's default stays scalar. + /// The implementation to call: the Keccak backend, followed only + /// when all generated callers' feature requirements are met (the + /// draft's default stays scalar), and AVX2 for the polynomial + /// arithmetic on x86-64 when the CPU has what its callers need. fn backend() -> Backend { #[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))] if !$crate::cpu::detected().contains($crate::cpu::Features::all(&[ @@ -59,7 +103,11 @@ macro_rules! ml_dsa { ])) { return Backend::Scalar; } - Backend::detected() + #[cfg(target_arch = "x86_64")] + const AVX2: &[&[&str]] = &[$keygen_avx2_features, $sign_avx2_features, $verify_avx2_features]; + #[cfg(not(target_arch = "x86_64"))] + const AVX2: &[&[&str]] = &[]; + Backend::select($crate::hashes::sha3::Backend::detected(), $crate::cpu::detected(), AVX2) } /// Why an operation failed. @@ -140,6 +188,8 @@ macro_rules! ml_dsa { Backend::Scalar => $verify(&self.bytes, mu, sig, &mut scratch), #[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))] Backend::Sha3 => $verify_sha3(&self.bytes, mu, sig, &mut scratch), + #[cfg(target_arch = "x86_64")] + Backend::Avx2 => $verify_avx2(&self.bytes, mu, sig, &mut scratch), } }; if r == 1 { Ok(()) } else { Err(Error::InvalidSignature) } @@ -196,6 +246,8 @@ macro_rules! ml_dsa { Backend::Scalar => $keygen(seed, &mut key.vk.bytes, &mut key.sk, &mut scratch), #[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))] Backend::Sha3 => $keygen_sha3(seed, &mut key.vk.bytes, &mut key.sk, &mut scratch), + #[cfg(target_arch = "x86_64")] + Backend::Avx2 => $keygen_avx2(seed, &mut key.vk.bytes, &mut key.sk, &mut scratch), } }; zeroize(&mut scratch); @@ -274,6 +326,8 @@ macro_rules! ml_dsa { Backend::Scalar => $sign(&self.sk, mu, rnd, &mut sig, &mut scratch), #[cfg(all(target_arch = "aarch64", feature = "cpu-features-env"))] Backend::Sha3 => $sign_sha3(&self.sk, mu, rnd, &mut sig, &mut scratch), + #[cfg(target_arch = "x86_64")] + Backend::Avx2 => $sign_avx2(&self.sk, mu, rnd, &mut sig, &mut scratch), } }; zeroize(&mut scratch); From b818911af618d647bdb2f66fcd1f4953adfa6857 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 17:32:53 +0000 Subject: [PATCH 5/8] Spec: ML-DSA's RejNTTPoly four times (vg_mldsa_rej_ntt_poly4) The contract of a function that samples four elements of the matrix A at once, as ML-KEM's vg_mlkem_sample_ntt4 does: for each of the four 34-byte seeds, RejNTTPoly (FIPS 204 Algorithm 30) of it, reduced, or 0 if the loop does not finish within Appendix C's least bound for one of them. Its scratch is that of vg_mldsa_rej_ntt_poly (256 u64s), so callers can pass the same working space. No implementation yet: an x86-64 one, with four SHAKE128 instances in AVX2 registers, follows in its own PR. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- lean/VerifiedGarbage/Spec/MlDsa/Poly.lean | 46 +++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/lean/VerifiedGarbage/Spec/MlDsa/Poly.lean b/lean/VerifiedGarbage/Spec/MlDsa/Poly.lean index c3610ab84..ed63c8ff9 100644 --- a/lean/VerifiedGarbage/Spec/MlDsa/Poly.lean +++ b/lean/VerifiedGarbage/Spec/MlDsa/Poly.lean @@ -182,6 +182,34 @@ def rejNTTContract {M : ISA} (A : Abi M) (stack : Nat := 0) : Contract M := (stack := stack) (leak := some fun seed _a _scratch m => leakBytes (bytesAt m seed 34)) +/-- `vg_mldsa_rej_ntt_poly4(seeds: *const [u8; 136], a: *mut [u32; 1024], scratch: *mut [u64; 256]) -> u32`. -/ +def rejNTT4Sig : Sig where + params := [("seeds", .array false .u8 136), ("a", .array true .u32 1024), + ("scratch", .array true .u64 256)] + ret := some .u32 + +/-- Seed `k` of four at `seeds`: the 34 bytes from byte `34 k`. -/ +def seed4 (m : Mem) (seeds : Addr) (k : Nat) : List Byte := bytesAt m (seeds + BitVec.ofNat 64 (34 * k)) 34 + +/-- Polynomial `k` of four at `a`: from byte `1024 k`. -/ +def poly4 (a : Addr) (k : Nat) : Addr := a + BitVec.ofNat 64 (1024 * k) + +/-- `RejNTTPoly` four times: with the four 34-byte seeds `ρ₀, …, ρ₃` at +`seeds` (`seed4`), writes `RejNTTPoly(ρₖ)` (Algorithm 30) to the polynomial +at `a + 1024 k` (`poly4`), reduced, for each `k`, and returns 1; or returns +0 if the loop of `RejNTTPoly` does not finish within the least bound +Appendix C allows (`minBounds`) for one of them, and `a` is unspecified. +May leak the seeds. -/ +def rejNTT4Contract {M : ISA} (A : Abi M) (stack : Nat := 0) : Contract M := + rejNTT4Sig.contract A + (post := fun seeds a _scratch m m' r => + (r = 1 → ∀ k < 4, Reduced m' (poly4 a k)) ∧ + ((r = 1 ∧ ∀ k < 4, ∃ b : Bounds, rejNTTPoly b.rejNTT (seed4 m seeds k) = some (polyAt m' (poly4 a k))) ∨ + (r = 0 ∧ ∃ k < 4, rejNTTPoly minBounds.rejNTT (seed4 m seeds k) = none))) + (writeArgs := true) + (stack := stack) + (leak := some fun seeds _a _scratch m => leakBytes (bytesAt m seeds 136)) + /-- Whether `RejBoundedPoly` accepts the half-byte `b`: 1 if `CoeffFromHalfByte(b)` (Algorithm 15) is not `⊥`, 0 if it is. -/ def halfByteOk (η b : Nat) : Nat := if (coeffFromHalfByte η b).isSome then 1 else 0 @@ -591,6 +619,24 @@ def rejNTTApi : Api where indices), but not on anything else." safety := [scratchSafety] +/-- `vg_mldsa_rej_ntt_poly4` on every target. -/ +def rejNTT4Api : Api where + module := "mldsa" + name := "vg_mldsa_rej_ntt_poly4" + sig := rejNTT4Sig + writeArgs := true + contracts := some fun A stack => rejNTT4Contract A stack + summary := "`RejNTTPoly` (FIPS 204 Algorithm 30) four times: for each `k` < 4, writes the \ + element of `T_q` sampled from the SHAKE128 output of the 34 bytes of `*seeds` from byte \ + `34 k` to the 256 coefficients of `*a` from coefficient `256 k` (each less than `q` = \ + 8380417), and returns 1. " ++ boundDoc "a" "894 bytes of SHAKE128 output for each" ++ " The \ + four are independent, so an implementation may compute them together (e.g. four SHAKE128 \ + instances at once in vector registers).\n\n\ + Contract: `VG.Spec.MlDsa.rejNTT4Contract`. Not constant time in the seeds: timing may \ + depend on the pointers and on `*seeds` (public in ML-DSA: the seed `ρ` of the matrix and \ + indices), but not on anything else." + safety := [scratchSafety] + /-- `vg_mldsa_rej_bounded_poly` on every target. -/ def rejBoundedApi : Api where module := "mldsa" From 779ca16ebf070c12dfad8f8b5d9538b22483bb49 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 17:33:52 +0000 Subject: [PATCH 6/8] Spec: give vg_mldsa_rej_ntt_poly4 8 KiB of scratch Four interleaved Keccak states, the second buffer of the 4-way permutation and its table of round constants already take 2368 bytes, before the squeezed output; ML-KEM's vg_mlkem_sample_ntt4 has 1024 u64s for the same. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- lean/VerifiedGarbage/Spec/MlDsa/Poly.lean | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lean/VerifiedGarbage/Spec/MlDsa/Poly.lean b/lean/VerifiedGarbage/Spec/MlDsa/Poly.lean index ed63c8ff9..002339b9b 100644 --- a/lean/VerifiedGarbage/Spec/MlDsa/Poly.lean +++ b/lean/VerifiedGarbage/Spec/MlDsa/Poly.lean @@ -182,10 +182,10 @@ def rejNTTContract {M : ISA} (A : Abi M) (stack : Nat := 0) : Contract M := (stack := stack) (leak := some fun seed _a _scratch m => leakBytes (bytesAt m seed 34)) -/-- `vg_mldsa_rej_ntt_poly4(seeds: *const [u8; 136], a: *mut [u32; 1024], scratch: *mut [u64; 256]) -> u32`. -/ +/-- `vg_mldsa_rej_ntt_poly4(seeds: *const [u8; 136], a: *mut [u32; 1024], scratch: *mut [u64; 1024]) -> u32`. -/ def rejNTT4Sig : Sig where params := [("seeds", .array false .u8 136), ("a", .array true .u32 1024), - ("scratch", .array true .u64 256)] + ("scratch", .array true .u64 1024)] ret := some .u32 /-- Seed `k` of four at `seeds`: the 34 bytes from byte `34 k`. -/ From e365dde0d669990101e894c15c77beef0084e10d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 17:35:15 +0000 Subject: [PATCH 7/8] CI: test ML-DSA on the CPUs of the x86-64 feature matrix ML-DSA now chooses between its SSE2 and AVX2 polynomial arithmetic by CPU feature, so test it end to end where the choice differs: under SDE's Pentium 4 (the SSE2 code, and the line that chooses it) and Haswell. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77d025c99..63fd87b9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -391,7 +391,8 @@ jobs: # * `skx` (Skylake-SP): AVX-512F, so ChaCha20 with AVX-512F, and SHA-256 # and SHA-512 with AVX2; and BMI2 and ADX, so X25519 with them; # * `hsw` (Haswell): AVX2 and BMI but not AVX-512F or ADX, so ChaCha20 - # with AVX2 even where this runner has AVX-512F, and X25519's baseline; + # with AVX2 even where this runner has AVX-512F, X25519's baseline, and + # ML-DSA with its polynomial arithmetic in AVX2; # * `p4p` (Pentium 4): nothing beyond x86-64's baseline, not even CPUID # leaf 7, so the baseline ISA's implementations everywhere; # * `arl` (Arrow Lake): the SHA512 extension, so SHA-512 with it, which no @@ -407,7 +408,7 @@ jobs: fail-fast: false matrix: chip: [native, skx, hsw, p4p] - tests: [sha1 sha256 sha384 sha512 aes_gcm cmac chacha20 mlkem x25519 ed25519 cpu] + tests: [sha1 sha256 sha384 sha512 aes_gcm cmac chacha20 mlkem mldsa x25519 ed25519 cpu] include: - chip: arl tests: sha384 sha512 ed25519 cpu From be2f80c6e5355e0f799c1674b82198b36f3563f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 18:59:35 +0000 Subject: [PATCH 8/8] =?UTF-8?q?ML-DSA=20on=20x86-64:=20sample=20=C3=82=20f?= =?UTF-8?q?our=20entries=20at=20a=20time=20in=20verification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vg_mldsa_rej_ntt_poly4 runs RejNTTPoly on four seeds. Its AVX2 variant runs the four SHAKE128 instances at once in the 64-bit elements of ymm registers, with the absorb and squeeze code of vg_mlkem_sample_ntt4_avx2, and runs vg_mldsa_rej_ntt_poly's loop over the same 1008 bytes of each seed's output (squeezed in two rounds of three blocks). The baseline calls vg_mldsa_rej_ntt_poly on each seed. Both are proven against rejNTT4Contract with 24 bytes of stack, and are a new function of the MlDsaArith backend. Verification now samples each row of  four entries at a time from SB4 (four copies of ρ with the row's indices): the entries from 0, then for ℓ = 7 the last four (sampling entry 3 again), or for ℓ = 5 the last one alone. Its rej4 working space is the 8 KiB after the last row of Â, which the scratch size already allows. Its stack grows from 24 to 32 bytes, as its baseline callee calls three deep. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- README.md | 6 +- docs/algorithms/ml-dsa-44.toml | 2 +- docs/algorithms/ml-dsa-65.toml | 2 +- docs/algorithms/ml-dsa-87.toml | 2 +- .../Artifacts/MlDsaSample/X86_64.lean | 23 + .../MlDsaArith/X86_64/MlDsaVerify.lean | 16 +- .../Impl/MlDsa/X86_64/Arith/Avx2.lean | 3 +- .../Impl/MlDsa/X86_64/Arith/Backend.lean | 10 +- .../Impl/MlDsa/X86_64/Sample/RejNtt4.lean | 85 + .../Impl/MlDsa/X86_64/Verify/Frag.lean | 26 +- .../Impl/MlDsa/X86_64/Verify/Verify.lean | 32 +- .../Proof/MlDsa/X86_64/Arith/Backend.lean | 15 +- .../Proof/MlDsa/X86_64/Arith/BackendAvx2.lean | 4 +- .../Proof/MlDsa/X86_64/Sample/Rej4CT.lean | 177 + .../Proof/MlDsa/X86_64/Sample/Rej4Parse.lean | 215 + .../Proof/MlDsa/X86_64/Sample/Rej4Scalar.lean | 264 + .../Proof/MlDsa/X86_64/Sample/Rej4Sq.lean | 156 + .../Proof/MlDsa/X86_64/Sample/Rej4Top.lean | 412 + .../MlDsa/X86_64/Sample/Rej4Verified.lean | 79 + .../Proof/MlDsa/X86_64/Sample/RejBounded.lean | 2 +- .../MlDsa/X86_64/Sample/RejBoundedCT.lean | 10 +- .../MlDsa/X86_64/Sample/RejBoundedLoop.lean | 6 +- .../Proof/MlDsa/X86_64/Sample/RejNtt.lean | 2 +- .../Proof/MlDsa/X86_64/Sample/RejNttCT.lean | 6 +- .../Proof/MlDsa/X86_64/Sample/RejNttLoop.lean | 17 +- .../Proof/MlDsa/X86_64/Verify/Base.lean | 16 +- .../Proof/MlDsa/X86_64/Verify/Blocks.lean | 39 +- .../Proof/MlDsa/X86_64/Verify/CTBase.lean | 17 +- .../Proof/MlDsa/X86_64/Verify/CTSample.lean | 122 +- .../Proof/MlDsa/X86_64/Verify/CTStages.lean | 7 +- .../Proof/MlDsa/X86_64/Verify/Call.lean | 4 +- .../Proof/MlDsa/X86_64/Verify/CallSample.lean | 90 + .../Proof/MlDsa/X86_64/Verify/Entry.lean | 37 +- .../Proof/MlDsa/X86_64/Verify/Flag.lean | 53 + .../Proof/MlDsa/X86_64/Verify/Hash.lean | 14 +- .../Proof/MlDsa/X86_64/Verify/HashCT.lean | 2 +- .../Proof/MlDsa/X86_64/Verify/Instrs.lean | 33 +- .../Proof/MlDsa/X86_64/Verify/Prims.lean | 7 +- .../Proof/MlDsa/X86_64/Verify/PrimsOk.lean | 7 +- .../Proof/MlDsa/X86_64/Verify/StageA.lean | 286 +- .../Proof/MlDsa/X86_64/Verify/Top.lean | 14 +- .../Proof/MlDsa/X86_64/Verify/Verified.lean | 8 +- src/asm/x86_64/mldsa.rs | 4841 ++++++++++++ src/asm/x86_64/mldsa44.rs | 816 +- src/asm/x86_64/mldsa65.rs | 1508 ++-- src/asm/x86_64/mldsa87.rs | 6838 +++++++---------- 46 files changed, 10367 insertions(+), 5964 deletions(-) create mode 100644 lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/RejNtt4.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4CT.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Parse.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Scalar.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Sq.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Top.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Verified.lean diff --git a/README.md b/README.md index e3c64b3bc..e187c126f 100644 --- a/README.md +++ b/README.md @@ -827,7 +827,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) ✅ SHA extensions @@ -843,7 +843,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) ✅ SHA extensions @@ -859,7 +859,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) ✅ SHA extensions diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml index 766d606c3..4f28100c0 100644 --- a/docs/algorithms/ml-dsa-44.toml +++ b/docs/algorithms/ml-dsa-44.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa44.rs"] asm = ["mldsa44", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" } diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml index 8835d083b..6683e2898 100644 --- a/docs/algorithms/ml-dsa-65.toml +++ b/docs/algorithms/ml-dsa-65.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa65.rs"] asm = ["mldsa65", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" } diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml index 9a82d4299..649298bea 100644 --- a/docs/algorithms/ml-dsa-87.toml +++ b/docs/algorithms/ml-dsa-87.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa87.rs"] asm = ["mldsa87", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" } diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean index ca47fca64..67b44002a 100644 --- a/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean +++ b/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean @@ -4,6 +4,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejNttCT import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejBoundedCT import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.ExpandMask import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.BallCT +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Verified /-! # ML-DSA (FIPS 204) on x86-64: the sampling primitives @@ -32,6 +33,28 @@ def artifacts : List Artifact := [ stack := 16 verified := Proof.MlDsa.X86_64.Sample.rejNTT_verified spSafe := Code.all_of_allInstrs (by lit_decide) }, + { Spec.MlDsa.rejNTT4Api with + target := X86_64.target + doc := Spec.MlDsa.rejNTT4Api.doc (notes := ["It calls `vg_mldsa_rej_ntt_poly` on each seed, and saves \ + its caller's callee-saved registers in `scratch`."]) + code := Impl.MlDsa.X86_64.Sample.Rej4.rejNTT4 + contract := Spec.MlDsa.rejNTT4Contract X86_64.abi 24 + stack := 24 + verified := Proof.MlDsa.X86_64.Rej4.rejNTT4_verified + spSafe := Code.all_of_allInstrs (by decide +kernel) }, + { Spec.MlDsa.rejNTT4Api with + name := Spec.MlDsa.rejNTT4Api.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.rejNTT4Api.doc (notes := ["It runs the four instances of SHAKE128 at once, in the four \ + 64-bit elements of AVX2 registers (as `vg_mlkem_sample_ntt4_avx2` does), squeezing three blocks of each \ + twice, and runs the loop of `RejNTTPoly` over the 1008 bytes of each seed's output, as \ + `vg_mldsa_rej_ntt_poly` does. It saves its caller's callee-saved registers in `scratch`."]) + code := Impl.MlDsa.X86_64.Sample.Rej4.rejNTT4Avx2 + contract := Spec.MlDsa.rejNTT4Contract X86_64.abi 24 + stack := 24 + verified := Proof.MlDsa.X86_64.Rej4.rejNTT4Avx2_verified + spSafe := Code.all_of_allInstrs (by decide +kernel) + features := ["avx", "avx2"] }, { Spec.MlDsa.rejBoundedApi with target := X86_64.target doc := Spec.MlDsa.rejBoundedApi.doc (notes := ["It squeezes 544 bytes of SHAKE256 output (4 blocks) and \ diff --git a/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean index 903d1f596..ec57e7834 100644 --- a/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean +++ b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean @@ -16,8 +16,8 @@ emitter adds the `# Safety` items that depend on the target (`Sig.layoutDoc`), from `stack` and `writeArgs`, which `ofSig` checks against the contract. -The stack is 24 bytes: the return address of a call of a primitive, and up -to 16 bytes for its own calls. +The stack is 32 bytes: the return address of a call of a primitive, and up +to 24 bytes for its own calls (`vg_mldsa_rej_ntt_poly4`'s). -/ namespace VG.Generic.MlDsaArith.X86_64.MlDsaVerify @@ -38,8 +38,8 @@ def artifacts (v : ArithImpl) : List Artifact := [ target := X86_64.target doc := Spec.MlDsa.verify44Api.doc (notes := [note]) code := Impl.MlDsa.X86_64.Verify.verify (primsWith v.code) Spec.MlDsa.mlDsa44 - contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa44 X86_64.abi 24 - stack := 24 + contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa44 X86_64.abi 32 + stack := 32 verified := verify_prims v (List.mem_cons_self ..) spSafe := verify_spSafe (prims_okWith v) (List.mem_cons_self ..) }, { Spec.MlDsa.verify65Api with @@ -48,8 +48,8 @@ def artifacts (v : ArithImpl) : List Artifact := [ target := X86_64.target doc := Spec.MlDsa.verify65Api.doc (notes := [note]) code := Impl.MlDsa.X86_64.Verify.verify (primsWith v.code) Spec.MlDsa.mlDsa65 - contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa65 X86_64.abi 24 - stack := 24 + contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa65 X86_64.abi 32 + stack := 32 verified := verify_prims v (List.mem_cons_of_mem _ (List.mem_cons_self ..)) spSafe := verify_spSafe (prims_okWith v) (List.mem_cons_of_mem _ (List.mem_cons_self ..)) }, { Spec.MlDsa.verify87Api with @@ -58,8 +58,8 @@ def artifacts (v : ArithImpl) : List Artifact := [ target := X86_64.target doc := Spec.MlDsa.verify87Api.doc (notes := [note]) code := Impl.MlDsa.X86_64.Verify.verify (primsWith v.code) Spec.MlDsa.mlDsa87 - contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa87 X86_64.abi 24 - stack := 24 + contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa87 X86_64.abi 32 + stack := 32 verified := verify_prims v (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))) spSafe := verify_spSafe (prims_okWith v) (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))) }] diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean index 4c4172ca7..3c987cbb6 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean @@ -198,6 +198,7 @@ def subAvx2 : Prog isa := .seq (.block (yconst .xmm15 8380417)) (.seq (rcxLoop 32 (yaccBody .psubd (vcadd .xmm0 .xmm2))) (.block yepi)) /-- The AVX2 code. -/ -def Backend.avx2 : Backend := ⟨nttAvx2, nttInvAvx2, mulAvx2, mulAddAvx2, addAvx2, subAvx2, "_avx2"⟩ +def Backend.avx2 : Backend := + ⟨nttAvx2, nttInvAvx2, mulAvx2, mulAddAvx2, addAvx2, subAvx2, Sample.Rej4.rejNTT4Avx2, "_avx2"⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean index c6ef074df..65d34d28d 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean @@ -1,6 +1,7 @@ import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Ntt import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Mul import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.AddSub +import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejNtt4 /-! # ML-DSA on x86-64: implementations of the polynomial arithmetic @@ -8,7 +9,8 @@ import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.AddSub Key generation, signing and verification call the polynomial arithmetic of one implementation, a `Backend`: the code of `vg_mldsa_ntt`, `vg_mldsa_inv_ntt`, `vg_mldsa_multiply_ntt`, `vg_mldsa_multiply_add_ntt`, -`vg_mldsa_add` and `vg_mldsa_sub`, whose names end with `sfx` (e.g. +`vg_mldsa_add` and `vg_mldsa_sub`, and of `vg_mldsa_rej_ntt_poly4` +(which samples four entries of the matrix `Â` at once), whose names end with `sfx` (e.g. `_avx2`; nothing for the SSE2 code, `sse2`). Each is a variant of the interface `MlDsaArith` on x86-64 (`Variants/MlDsaArith/X86_64/`), and the functions that call them are emitted once for each @@ -27,15 +29,17 @@ structure Backend where mulAdd : Prog isa add : Prog isa sub : Prog isa + rej4 : Prog isa /-- What the names of its functions, and of those calling them, end with. -/ sfx : String /-- The SSE2 code. -/ -def Backend.sse2 : Backend := ⟨Arith.ntt, Arith.nttInv, Arith.mul, Arith.mulAdd, Arith.add, Arith.sub, ""⟩ +def Backend.sse2 : Backend := + ⟨Arith.ntt, Arith.nttInv, Arith.mul, Arith.mulAdd, Arith.add, Arith.sub, Sample.Rej4.rejNTT4, ""⟩ /-- Every function empty, which the proofs that the functions calling a backend never write `rsp` (and load MXCSR only to restore it) evaluate in its place. -/ -def Backend.empty : Backend := ⟨.block [], .block [], .block [], .block [], .block [], .block [], ""⟩ +def Backend.empty : Backend := ⟨.block [], .block [], .block [], .block [], .block [], .block [], .block [], ""⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/RejNtt4.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/RejNtt4.lean new file mode 100644 index 000000000..a13540e9d --- /dev/null +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/RejNtt4.lean @@ -0,0 +1,85 @@ +import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejNtt +import VerifiedGarbage.Impl.MlKem.X86_64.Sample4 + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4` and `vg_mldsa_rej_ntt_poly4_avx2` + +`rejNTT4(seeds = rdi, a = rsi, scratch = rdx) -> eax` runs `RejNTTPoly` on +four seeds. The baseline implementation (`rejNTT4`) calls +`vg_mldsa_rej_ntt_poly` on each, with the prologue and epilogue of +`vg_mlkem_sample_ntt4` (`Impl/MlKem/X86_64/Sample4.lean`: `scratch` in +`rbx`, `seeds` in `r12`, `a` in `r13`, the AND of the results in `r14`, and +their caller's values and `rbp`'s saved in `scratch[4384..4424)`) and its +scratch space from byte 6144 of `scratch`. + +The one for AVX2 (`rejNTT4Avx2`) runs the four SHAKE128 instances at once +in the four 64-bit elements of `ymm` registers, as `vg_mlkem_sample_ntt4_avx2` +does, with the same code to absorb the seeds and squeeze three blocks of +each (504 bytes, from byte `2368 + 504 k` of `scratch`). It samples from them +with the loop of `vg_mldsa_rej_ntt_poly` (`rnBody`, 168 iterations of 3 +bytes), keeping the number `j` of coefficients of seed `k` at +`scratch[4424 + 8 k]`; squeezes three more blocks of each to the same +place, and runs 168 more iterations of the loop on them. That is the loop of +`vg_mldsa_rej_ntt_poly` on the same 1008 bytes of output, so seed `k` has +256 coefficients if and only if `vg_mldsa_rej_ntt_poly` would have them +(and if not, neither has `RejNTTPoly` within the least bound of FIPS 204 +Appendix C, 894 bytes). It returns 1 if every seed has 256 coefficients +(the AND of `j >> 8`), and 0 otherwise. + +The loops' branches and the addresses of their stores depend on the XOF +output, a function of the seeds, and on nothing else; every other address +and branch depends only on the pointers. +-/ + +namespace VG.Impl.MlDsa.X86_64.Sample.Rej4 + +open VG.X86_64 +open VG.Impl.MlKem.X86_64 (at_) +open VG.Impl.MlKem.X86_64.Sample4 (absorb4 squeeze4 oRc oBuf oScalar) +open VG.Impl.Sha3.X86_64.X4 (rcTable) + +/-- Where `j` of seed `k` is kept, at `scratch + oJ + 8 k`. -/ +def oJ : Nat := 4424 + +/-- `j ← 0` for each seed. -/ +def zeroJ : List Instr := + .mov32 .rax (.imm 0) :: (List.range 4).map fun k => .store (at_ .rbx (oJ + 8 * k)) .rax + +/-- 168 iterations of `RejNTTPoly`'s loop on the 504 bytes of the output of +seed `k`, to polynomial `k`, from `j` at `scratch + oJ + 8 k`. -/ +def half (k : Nat) : Prog isa := + .seq (.block [.mov .rsi (.reg .rbx), .alu .add .rsi (.imm (BitVec.ofNat 32 (oBuf + 504 * k))), + .mov .rbp (.reg .r13), .alu .add .rbp (.imm (BitVec.ofNat 32 (1024 * k))), + .mov .rdi (.mem (at_ .rbx (oJ + 8 * k))), .mov32 .rcx (.imm 168)]) + (.loop rnBody .ne) + +/-- The first half of seed `k`, and `j` kept. -/ +def first (k : Nat) : Prog isa := .seq (half k) (.block [.store (at_ .rbx (oJ + 8 * k)) .rdi]) + +/-- The second half of seed `k`, and `r14 ← r14 ∧ (j >> 8)`. -/ +def second (k : Nat) : Prog isa := + .seq (half k) (.block [.mov .rax (.reg .rdi), .shift .shr .rax 8, .alu32 .and .r14 (.reg .rax)]) + +/-- `vg_mldsa_rej_ntt_poly4_avx2`. `vzeroupper` clears the upper halves of +the vector registers after the last permutation. -/ +def rejNTT4Avx2 : Prog isa := + .seq (.block (VG.Impl.MlKem.X86_64.Sample4.pro ++ rcTable .rbx (oRc / 32) ++ absorb4)) + (.seq (squeeze4 0) (.seq (squeeze4 1) (.seq (squeeze4 2) (.seq (.block zeroJ) + (.seq (first 0) (.seq (first 1) (.seq (first 2) (.seq (first 3) + (.seq (squeeze4 0) (.seq (squeeze4 1) (.seq (squeeze4 2) (.seq (.block [.vop .vzeroupper]) + (.seq (second 0) (.seq (second 1) (.seq (second 2) (.seq (second 3) (.block VG.Impl.MlKem.X86_64.Sample4.epi))))))))))))))))) + +/-- `vg_mldsa_rej_ntt_poly` on seed `k`, and `r14 ← r14 ∧ result`. -/ +def callK (k : Nat) : Prog isa := + .seq (.block [.mov .rdi (.reg .r12), .alu .add .rdi (.imm (BitVec.ofNat 32 (34 * k))), + .mov .rsi (.reg .r13), .alu .add .rsi (.imm (BitVec.ofNat 32 (1024 * k))), .mov .rdx (.reg .rbx), + .alu .add .rdx (.imm (BitVec.ofNat 32 oScalar))]) + (.seq (.call "vg_mldsa_rej_ntt_poly" rejNTT) (.block [.alu32 .and .r14 (.reg .rax)])) + +/-- `vg_mldsa_rej_ntt_poly4`: `vg_mldsa_rej_ntt_poly` on each seed, with the +prologue and epilogue of `vg_mldsa_rej_ntt_poly4_avx2`. -/ +def rejNTT4 : Prog isa := + .seq (.block VG.Impl.MlKem.X86_64.Sample4.pro) (.seq (callK 0) (.seq (callK 1) (.seq (callK 2) (.seq (callK 3) + (.block VG.Impl.MlKem.X86_64.Sample4.epi))))) + +end VG.Impl.MlDsa.X86_64.Sample.Rej4 diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean index 5b4e5e24f..72ef6c175 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean @@ -21,11 +21,13 @@ The layout of `scratch` (in bytes): the Keccak state at 0 (200 bytes) and the sponge functions' working space at 200 (640 bytes); the caller's callee-saved registers at 840 (48 bytes); the seed of `RejNTTPoly` (`SB`, 34 bytes) at 896; `w1Encode(w′₁)` at 1024 (at most 1024 bytes); the -recomputed commitment hash `c̃′` at 2048 (at most 64 bytes); the working +recomputed commitment hash `c̃′` at 2048 (at most 64 bytes); the four seeds +of `vg_mldsa_rej_ntt_poly4` (`SB4`, 136 bytes) at 2560; the working space of the primitives at 4096 (2048 bytes); and polynomials of 1024 bytes from 8192 (`P j`): the hint `h` (polynomials 0 to 7, of which the first `k`), `z` (8 to 14), `c` (15), two temporaries (16, 17), `w′` (18), `w′₁` -(19), and `Â[r, s]` (`20 + 8r + s`). +(19), and `Â[r, s]` (`20 + 8r + s`), then the working space of +`vg_mldsa_rej_ntt_poly4` (8 KiB, after the last row of `Â`). -/ namespace VG.Impl.MlDsa.X86_64.Verify @@ -42,6 +44,7 @@ def oSV : Nat := 840 def oSB : Nat := 896 def oB : Nat := 1024 def oCT : Nat := 2048 +def oSB4 : Nat := 2560 def oSS : Nat := 4096 /-- Polynomial `j`. -/ def oP (j : Nat) : Nat := 8192 + 1024 * j @@ -137,6 +140,7 @@ structure Prims where unpackT1 : Prog isa hintUnpack : Prog isa normLt : Prog isa + rej4 : Prog isa /-- What the names of the polynomial arithmetic's functions end with (`Arith.Backend`). -/ sfx : String := "" @@ -159,6 +163,9 @@ def subAt (f g : Ptr) : Prog isa := callAt ("vg_mldsa_sub" ++ P.sfx) P.sub [(.rd def rejNttAt (a : Ptr) : Prog isa := callAt "vg_mldsa_rej_ntt_poly" P.rejNtt [(.rdi, .ptr (sc oSB)), (.rsi, .ptr a), (.rdx, .ptr (sc oSS))] +def rej4At (a w : Ptr) : Prog isa := + callAt ("vg_mldsa_rej_ntt_poly4" ++ P.sfx) P.rej4 [(.rdi, .ptr (sc oSB4)), (.rsi, .ptr a), (.rdx, .ptr w)] + def ballAt (ct : Ptr) (len tau : Nat) (c : Ptr) : Prog isa := callAt "vg_mldsa_sample_in_ball" P.ball [(.rdi, .ptr ct), (.rsi, .imm len), (.rdx, .imm tau), (.rcx, .ptr c), (.r8, .ptr (sc oSS))] @@ -191,12 +198,13 @@ end /-- `r15 ← r15 ∧ eax`. -/ def and15 : List Instr := [.alu32 .and .r15 (.reg .rax)] -/-- The polynomial at `a` masked by the result `eax` (0 or 1) of the -sampler that wrote it: unchanged if 1, and zero if 0, so that it is reduced -either way, without a branch. `edx ← -eax`, then each coefficient `∧ edx`. -/ -def mask (a : Ptr) : Prog isa := +/-- The polynomial at `a` (or the `N` coefficients from `a`) masked by the +result `eax` (0 or 1) of the sampler that wrote it: unchanged if 1, and zero +if 0, so that it is reduced either way, without a branch. `edx ← -eax`, then +each coefficient `∧ edx`. -/ +def mask (a : Ptr) (N : Nat := 256) : Prog isa := .seq (.block ([.mov32 .rdx (.imm 0), .alu32 .sub .rdx (.reg .rax)] ++ - glue [(.rdi, .ptr a), (.rcx, .imm 256)])) + glue [(.rdi, .ptr a), (.rcx, .imm N)])) (.loop (.block [.mov32 .rax (.mem (at_ .rdi 0)), .alu32 .and .rax (.reg .rdx), .store32 (at_ .rdi 0) .rax, .alu .add .rdi (.imm 4), .alu .sub .rcx (.imm 1)]) .ne) @@ -204,6 +212,10 @@ def mask (a : Ptr) : Prog isa := def sampled (call : Prog isa) (a : Ptr) : Prog isa := .seq call (.seq (.block and15) (mask a)) +/-- The same for a call that samples the four polynomials from `a`. -/ +def sampled4 (call : Prog isa) (a : Ptr) : Prog isa := + .seq call (.seq (.block and15) (mask a 1024)) + /-- `c` if `r15 ≠ 0`. -/ def ifOk (c : Prog isa) : Prog isa := .seq (.block [.alu32 .test .r15 (.reg .r15)]) (.ite .ne c (.block [])) diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean index b985e6cf0..4ef3f67af 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean @@ -16,8 +16,11 @@ in `scratch`. returns 0 at once if the hint is malformed. 2. `z[i] = BitUnpack` of the `i`-th piece of `σ` (polynomial `8 + i`), and `r15 ← r15 ∧ (‖z[i]‖∞ < γ₁ - β)`; it returns 0 if one of them is not. -3. `ρ` (`pk[0 : 32]`) to `SB`, and `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)` - (polynomial `20 + 8r + s`); `c = SampleInBall(c̃)` (polynomial 15). Each +3. `ρ` (`pk[0 : 32]`) to `SB` and to each seed of `SB4`, and + `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)` (polynomial `20 + 8r + s`), four + entries of a row at a time (`vg_mldsa_rej_ntt_poly4`): those from 0, then + for `ℓ = 7` those from 3 (sampling entry 3 again), or for `ℓ = 5` entry 4 + alone; `c = SampleInBall(c̃)` (polynomial 15). Each sampler's result is ANDed into `r15`, and its output masked with it (`sampled`): a sampler that fails leaves its output unspecified, and masking makes it reduced (zero) without a branch on the result, which @@ -73,12 +76,29 @@ def aOne (e : Nat) : Prog isa := .seq (.block (setB (sc (oSB + 32)) (e % 8) ++ setB (sc (oSB + 33)) (e / 8))) (sampled (rejNttAt P (pS (20 + e))) (pS (20 + e))) -/-- The entries `8r + s` of row `r` of `Â`. -/ -def aRow (r : Nat) : Prog isa := seqR (aOne P) (8 * r) p.ℓ +/-- Where `vg_mldsa_rej_ntt_poly4` works: after the last row of `Â`. -/ +def oR4 : Nat := oP (20 + 8 * p.k) -/-- `ρ` to `SB`, `Â`, and `c`. -/ +/-- The bytes `s ‖ r` of seed `k` of `SB4`. -/ +def setSR (r s k : Nat) : List Instr := setB (sc (oSB4 + 34 * k + 32)) (s + k) ++ setB (sc (oSB4 + 34 * k + 33)) r + +/-- `Â[r, s], …, Â[r, s + 3]`. -/ +def aGrp (r s : Nat) : Prog isa := + .seq (.block (setSR r s 0)) (.seq (.block (setSR r s 1)) (.seq (.block (setSR r s 2)) (.seq (.block (setSR r s 3)) + (sampled4 (rej4At P (pA r s) (sc (oR4 p))) (pA r s))))) + +/-- The entries of row `r` of `Â`: four from 0, then the last four if `ℓ = 7`, or the last one if `ℓ = 5`. -/ +def aRow (r : Nat) : Prog isa := + .seq (aGrp P p r 0) (if p.ℓ = 7 then aGrp P p r 3 else seqR (aOne P) (8 * r + 4) (p.ℓ - 4)) + +/-- `ρ` to `SB` and to the four seeds of `SB4`. -/ +def rhos : Prog isa := + .seq (copy (sc oSB) (.rbp, 0) 32) (.seq (copy (sc oSB4) (.rbp, 0) 32) (.seq (copy (sc (oSB4 + 34)) (.rbp, 0) 32) + (.seq (copy (sc (oSB4 + 68)) (.rbp, 0) 32) (copy (sc (oSB4 + 102)) (.rbp, 0) 32)))) + +/-- `ρ` to `SB` and `SB4`, `Â`, and `c`. -/ def samples : Prog isa := - .seq (copy (sc oSB) (.rbp, 0) 32) (.seq (seqR (aRow P p) 0 p.k) + .seq rhos (.seq (seqR (aRow P p) 0 p.k) (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC)) /-- `Σₛ Â[r, s] ẑ[s]` to `W`. -/ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean index 6a27d6c37..d8ad5b2bc 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean @@ -4,6 +4,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub import VerifiedGarbage.Proof.MlKem.X86_64.ArithOk +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Verified /-! # ML-DSA on x86-64: what the callers of the polynomial arithmetic need of it @@ -31,6 +32,15 @@ structure FnOk (k : Nat → Contract isa) (c : Prog isa) : Prop where ctl : ctlOk c = true sp : c.all (fun i => !isa.writesSp i) = true +/-- What a caller needs of `vg_mldsa_rej_ntt_poly4`, which calls three deep +and takes 24 bytes of stack. -/ +structure Rej4Ok (c : Prog isa) : Prop where + ver : Verified X86_64.target c (Spec.MlDsa.rejNTT4Contract X86_64.abi 24) + nosp : NoSp c + depth : c.depth ≤ 3 + ctl : ctlOk c = true + sp : c.all (fun i => !isa.writesSp i) = true + /-- Each function of the backend `B` meets its contract, and is safe to call. -/ structure BackendOk (B : Backend) : Prop where ntt : FnOk (fun S => Spec.MlDsa.nttContract X86_64.abi S) B.ntt @@ -39,6 +49,7 @@ structure BackendOk (B : Backend) : Prop where mulAdd : FnOk (fun S => Spec.MlDsa.mulAddContract X86_64.abi S) B.mulAdd add : FnOk (fun S => Spec.MlDsa.addContract X86_64.abi S) B.add sub : FnOk (fun S => Spec.MlDsa.subContract X86_64.abi S) B.sub + rej4 : Rej4Ok B.rej4 /-- An implementation of the polynomial arithmetic on x86-64. -/ structure ArithImpl where @@ -68,7 +79,9 @@ def ArithImpl.sse2 : ArithImpl where add := FnOk.of Arith.add_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) sub := FnOk.of Arith.sub_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) - (by decide +kernel) } + (by decide +kernel) + rej4 := ⟨Rej4.rejNTT4_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), by decide +kernel, + by decide +kernel, Code.all_of_allInstrs (by decide +kernel)⟩ } features := [] end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean index 157073d12..0036f4cdb 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean @@ -30,7 +30,9 @@ def ArithImpl.avx2 : ArithImpl where add := FnOk.of Arith.addY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) sub := FnOk.of Arith.subY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) - (by decide +kernel) } + (by decide +kernel) + rej4 := ⟨Rej4.rejNTT4Avx2_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), by decide +kernel, + by decide +kernel, Code.all_of_allInstrs (by decide +kernel)⟩ } features := ["avx", "avx2"] end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4CT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4CT.lean new file mode 100644 index 000000000..fe0aa67dc --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4CT.lean @@ -0,0 +1,177 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Top +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejNttCT +import VerifiedGarbage.Proof.MlKem.X86_64.S4CT + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4_avx2`, constant time but for the seeds + +Untrusted: everything here is checked by Lean. Two runs whose seeds (the +declared leak) and pointers agree leak the same. The code but for the loops +of the halves is proven by the taint analysis, from the pointers (the +prologue, the absorption and the first squeeze as in +`vg_mlkem_sample_ntt4_avx2`, `S4.start_ct`). Both runs read the same XOF +output, so in each half's loop they are at the same iteration with the same +coefficients sampled, and each iteration runs as in +`vg_mldsa_rej_ntt_poly` (`RejNttCT.body_ct`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Impl.MlDsa.X86_64.Sample (rnBody) +open VG.Impl.MlDsa.X86_64.Sample.Rej4 (oJ half first second zeroJ rejNTT4Avx2) +open VG.Proof.MlKem.X86_64 (sample4K relInv taintRel relStart Rel2 ofNat64_pred ofNat64_beq_zero RelCT.postDep) +open VG.Proof.MlKem.X86_64.S4 (R4 Pre aP at' pre_of pub_scr pub_aP pub_B env_rbx start_ct) +open VG.Proof.MlDsa.X86_64.Sample (nil_regs WP.all') +open VG.Proof.MlDsa.X86_64.Sample.RejNttCT (BPre BRel body_ct) +open VG.Spec.MlKem (poly4) + +theorem pub_Lt4 {σ₁ σ₂ : State} (hq : sample4K.pub σ₁ σ₂) {K : Nat} (hK : K < 4) (t : Nat) : + Lt σ₁ K t = Lt σ₂ K t := by simp only [Lt, Xb, pub_B hq hK] + +/-! ## The squeezes -/ + +theorem sqTaint0 : ∃ hc : VG.Taint.Hint X86_64.Taint.T, + (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 0) hc).isSome = true := ⟨_, by taint_decide⟩ +theorem sqTaint1 : ∃ hc : VG.Taint.Hint X86_64.Taint.T, + (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 1) hc).isSome = true := ⟨_, by taint_decide⟩ +theorem sqTaint2 : ∃ hc : VG.Taint.Hint X86_64.Taint.T, + (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 2) hc).isSome = true := ⟨_, by taint_decide⟩ + +/-- A squeeze, from the absorption, given its taint analysis. -/ +theorem sqT_ct {t n : Nat} (hn : n < 3) + (c : ∃ hc : VG.Taint.Hint X86_64.Taint.T, (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 n) hc).isSome = true) : + RelCT isa (R4 fun σ s => SqT σ t n s) (squeeze4 n) (R4 fun σ s => SqT σ t (n + 1) s) := by + obtain ⟨_, c⟩ := c + exact relInv (fun σ s hp h => WP.mono (sqT_ok (pre_of hp) hn h) fun _ h' => h'.1) + (taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) c) + +/-- A second squeeze, given its taint analysis. -/ +theorem sqM_ct {n : Nat} (hn : n < 3) + (c : ∃ hc : VG.Taint.Hint X86_64.Taint.T, (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 n) hc).isSome = true) : + RelCT isa (R4 fun σ s => M2 σ n s) (squeeze4 n) (R4 fun σ s => M2 σ (n + 1) s) := by + obtain ⟨_, c⟩ := c + exact relInv (fun σ s hp h => sqM_ok (pre_of hp) hn h) + (taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.sq.env h₂.sq.env) c) + +/-! ## The halves -/ + +/-- Two runs at iteration `t` of a half of seed `K`, from states with `X`, +`n = 168 - t` iterations from its end. -/ +def HI (X : State → State → Prop) (K h n : Nat) (s₁ s₂ : State) : Prop := + ∃ σ₁ σ₂ s₀₁ s₀₂ t, sample4K.pre σ₁ ∧ sample4K.pre σ₂ ∧ sample4K.pub σ₁ σ₂ ∧ n = 168 - t ∧ t < 168 ∧ + X σ₁ s₀₁ ∧ X σ₂ s₀₂ ∧ HAt σ₁ s₀₁ K h t s₁ ∧ HAt σ₂ s₀₂ K h t s₂ + +theorem hbpre {σ : State} (hp : Pre σ) {s₀ : State} {K h t : Nat} (hK : K < 4) (ht : t < 168) {s : State} + (hI : HAt σ s₀ K h t s) : BPre s (poly4 (aP σ) K) (Lt σ K (168 * h + t)) := + ⟨hI.rbp, hI.rdi, Lt_length_le σ K _, coeffsWr hp hK hI.env, hI.stored, + by simpa using hat_regions hp hK hI (j := 0) (by omega), hat_regions hp hK hI (by omega), + hat_regions hp hK hI (by omega)⟩ + +theorem hi_brel {X : State → State → Prop} {K h n : Nat} (hK : K < 4) (hh : h < 2) {s₁ s₂ : State} + (H : HI X K h n s₁ s₂) : BRel s₁ s₂ := by + obtain ⟨σ₁, σ₂, s₀₁, s₀₂, t, p₁, p₂, hq, _, ht, _, _, l₁, l₂⟩ := H + refine ⟨poly4 (aP σ₁) K, Lt σ₁ K (168 * h + t), hbpre (pre_of p₁) hK ht l₁, + by rw [pub_aP hq, pub_Lt4 hq hK]; exact hbpre (pre_of p₂) hK ht l₂, + by rw [l₁.rsi, l₂.rsi, at', at', pub_scr hq], by rw [l₁.rcx, l₂.rcx], fun k hk => ?_⟩ + rw [hat_byte hh l₁ (by omega), hat_byte hh l₂ (by omega)] + simp only [Xb, pub_B hq hK] + +/-- The loop of a half. -/ +theorem hloop_ct {X : State → State → Prop} {K h : Nat} (hK : K < 4) (hh : h < 2) (n : Nat) : + RelCT isa (HI X K h n) (.loop rnBody .ne) (R4 fun σ s => ∃ s₀, X σ s₀ ∧ HAt σ s₀ K h 168 s) := by + refine RelCT.loop (M := isa) (HI X K h) (fun n => ?_) n + refine RelCT.postDep (F := fun (x x' : State) => ∀ p : State × State × Nat, sample4K.pre p.1 ∧ p.2.2 < 168 ∧ + HAt p.1 p.2.1 K h p.2.2 x → + HAt p.1 p.2.1 K h (p.2.2 + 1) x' ∧ x'.zf = some (BitVec.ofNat 64 (168 - p.2.2) - 1 == 0)) + (RelCT.mono body_ct (fun x y H => hi_brel hK hh H) fun _ _ _ => trivial) (fun x y H => ?_) ?_ + · obtain ⟨σ₁, σ₂, s₀₁, s₀₂, t, p₁, p₂, _, _, ht, _, _, l₁, l₂⟩ := H + exact ⟨WP.all' (fun p hp' => hat_step (pre_of hp'.1) hK hh hp'.2.1 hp'.2.2) ⟨(σ₁, s₀₁, t), p₁, ht, l₁⟩, + WP.all' (fun p hp' => hat_step (pre_of hp'.1) hK hh hp'.2.1 hp'.2.2) ⟨(σ₂, s₀₂, t), p₂, ht, l₂⟩⟩ + · intro x y x' y' ⟨σ₁, σ₂, s₀₁, s₀₂, t, p₁, p₂, hq, hn, ht, x₁, x₂, l₁, l₂⟩ f₁ f₂ + obtain ⟨l₁', z₁⟩ := f₁ (σ₁, s₀₁, t) ⟨p₁, ht, l₁⟩ + obtain ⟨l₂', z₂⟩ := f₂ (σ₂, s₀₂, t) ⟨p₂, ht, l₂⟩ + have ez : (BitVec.ofNat 64 (168 - t) - 1 == 0) = decide (t + 1 = 168) := by + rw [ofNat64_pred (by omega) (by omega), ofNat64_beq_zero (by omega)] + exact decide_eq_decide.mpr (by omega) + rw [ez] at z₁ z₂ + refine ⟨by show x'.zf.map _ = y'.zf.map _; rw [z₁, z₂], fun hf => ?_, fun ht' => ?_⟩ + · have : t + 1 = 168 := by + have : x'.zf.map (!·) = some false := hf + rw [z₁] at this; simpa using this + rw [this] at l₁' l₂' + exact ⟨σ₁, σ₂, p₁, p₂, hq, ⟨s₀₁, x₁, l₁'⟩, ⟨s₀₂, x₂, l₂'⟩⟩ + · have : t + 1 ≠ 168 := by + have : x'.zf.map (!·) = some true := ht' + rw [z₁] at this; simpa using this + exact ⟨168 - (t + 1), by omega, σ₁, σ₂, s₀₁, s₀₂, t + 1, p₁, p₂, hq, rfl, by omega, x₁, x₂, l₁', l₂'⟩ + +/-- The setup of a half. -/ +abbrev hsetup (K : Nat) : List Instr := + [.mov .rsi (.reg .rbx), .alu .add .rsi (.imm (BitVec.ofNat 32 (oBuf + 504 * K))), + .mov .rbp (.reg .r13), .alu .add .rbp (.imm (BitVec.ofNat 32 (1024 * K))), + .mov .rdi (.mem (at_ .rbx (oJ + 8 * K))), .mov32 .rcx (.imm 168)] + +/-- A half of seed `K` from states with `X`, given the taint analysis of its setup. -/ +theorem half_ct {X : State → State → Prop} {K h : Nat} (hK : K < 4) (hh : h < 2) + (hX : ∀ σ s, sample4K.pre σ → X σ s → HPre σ K h s) {hc : VG.Taint.Hint X86_64.Taint.T} + (c : (taint.check (X86_64.Taint.ofRegs [.rbx, .r13]) (.block (hsetup K)) hc).isSome = true) : + RelCT isa (R4 X) (half K) (R4 fun σ s => ∃ s₀, X σ s₀ ∧ HAt σ s₀ K h 168 s) := by + unfold half + refine RelCT.seq (RelCT.mono (relInv (I' := fun σ s => ∃ s₀, X σ s₀ ∧ HAt σ s₀ K h 0 s) + (fun σ s hp hx => WP.mono (hsetup_ok (pre_of hp) hK (hX σ s hp hx)) fun _ h' => ⟨s, hx, h'⟩) + (taintRel [.rbx, .r13] (fun x y ⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩ r hr => by + have e₁ := (hX σ₁ x p₁ h₁).env + have e₂ := (hX σ₂ y p₂ h₂).env + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl + · exact env_rbx hq e₁ e₂ + · rw [e₁.r13, e₂.r13, pub_aP hq]) c)) (fun _ _ h => h) + (Q' := HI X K h 168) fun _ _ ⟨σ₁, σ₂, p₁, p₂, hq, ⟨s₀₁, x₁, l₁⟩, ⟨s₀₂, x₂, l₂⟩⟩ => + ⟨σ₁, σ₂, s₀₁, s₀₂, 0, p₁, p₂, hq, rfl, by decide, x₁, x₂, l₁, l₂⟩) (hloop_ct hK hh 168) + +/-- The first half of seed `K`, given the taint analysis of its blocks. -/ +theorem first_ct {K : Nat} (hK : K < 4) {h₁ h₂ : VG.Taint.Hint X86_64.Taint.T} + (c₁ : (taint.check (X86_64.Taint.ofRegs [.rbx, .r13]) (.block (hsetup K)) h₁).isSome = true) + (c₂ : (taint.check (X86_64.Taint.ofRegs [.rbx]) (.block [.store (at_ .rbx (oJ + 8 * K)) .rdi]) h₂).isSome = + true) : + RelCT isa (R4 fun σ s => P1 σ K s) (first K) (R4 fun σ s => P1 σ (K + 1) s) := + RelCT.seq (half_ct hK (by decide) (fun _ _ _ h => hpre1 hK h) c₁) + (relInv (fun σ s hp ⟨_, h₀, hA⟩ => firstEnd_ok (pre_of hp) hK h₀ hA) + (taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, ⟨_, _, a₁⟩, ⟨_, _, a₂⟩⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq a₁.env a₂.env) c₂)) + +/-- The second half of seed `K`, given the taint analysis of its setup. -/ +theorem second_ct {K : Nat} (hK : K < 4) {h₁ : VG.Taint.Hint X86_64.Taint.T} + (c₁ : (taint.check (X86_64.Taint.ofRegs [.rbx, .r13]) (.block (hsetup K)) h₁).isSome = true) : + RelCT isa (R4 fun σ s => P2 σ K s) (second K) (R4 fun σ s => P2 σ (K + 1) s) := + RelCT.seq (half_ct hK (by decide) (fun _ _ _ h => hpre2 hK h) c₁) + (relInv (fun σ s hp ⟨_, h₀, hA⟩ => secondEnd_ok (pre_of hp) hK h₀ hA) (taintRel [] nil_regs (by taint_decide))) + +/-! ## The whole function -/ + +theorem ct : ConstantTime isa r4K.pre r4K.pub rejNTT4Avx2 := by + refine relStart (Q := fun _ _ => True) (RelCT.seq start_ct (RelCT.seq (RelCT.mono (sqT_ct (t := 0) (by decide) sqTaint0) + (fun _ _ ⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩ => ⟨σ₁, σ₂, p₁, p₂, hq, sqT_of h₁, sqT_of h₂⟩) fun _ _ h => h) + (RelCT.seq (sqT_ct (by decide) sqTaint1) (RelCT.seq (sqT_ct (by decide) sqTaint2) ?_)))) + refine RelCT.seq (relInv (fun σ s hp h => zeroJ_ok (pre_of hp) h) + (taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) (by taint_decide))) ?_ + refine RelCT.seq (first_ct (K := 0) (by decide) (by taint_decide) (by taint_decide)) ?_ + refine RelCT.seq (first_ct (K := 1) (by decide) (by taint_decide) (by taint_decide)) ?_ + refine RelCT.seq (first_ct (K := 2) (by decide) (by taint_decide) (by taint_decide)) ?_ + refine RelCT.seq (first_ct (K := 3) (by decide) (by taint_decide) (by taint_decide)) ?_ + refine RelCT.seq (RelCT.mono (sqM_ct (by decide) sqTaint0) + (fun _ _ ⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩ => ⟨σ₁, σ₂, p₁, p₂, hq, m2_of h₁, m2_of h₂⟩) fun _ _ h => h) + (RelCT.seq (sqM_ct (by decide) sqTaint1) (RelCT.seq (sqM_ct (by decide) sqTaint2) ?_)) + refine RelCT.seq (relInv (fun σ s _ h => vz_ok h) (taintRel [] nil_regs (by taint_decide))) ?_ + refine RelCT.seq (second_ct (K := 0) (by decide) (by taint_decide)) ?_ + refine RelCT.seq (second_ct (K := 1) (by decide) (by taint_decide)) ?_ + refine RelCT.seq (second_ct (K := 2) (by decide) (by taint_decide)) ?_ + refine RelCT.seq (second_ct (K := 3) (by decide) (by taint_decide)) ?_ + exact taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) (by taint_decide) + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Parse.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Parse.lean new file mode 100644 index 000000000..da11e4f01 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Parse.lean @@ -0,0 +1,215 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Sq +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejNtt + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4_avx2`, sampling + +Untrusted: everything here is checked by Lean. Each half of a seed's +sampling (`half`) runs 168 iterations of `vg_mldsa_rej_ntt_poly`'s loop +(`rnBody_ok`) on the 504 bytes of the seed's buffer, which hold bytes +`504 h` to `504 h + 503` of its output `Xb` (`G` of the seed, as +`vg_mldsa_rej_ntt_poly` squeezes it: `Xb_getD`): from `j` coefficients +sampled, it samples those of the first `504 (h + 1)` bytes (`half_ok`), and +writes only the seed's polynomial. +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Impl.MlDsa.X86_64.Sample (rnBody) +open VG.Impl.MlDsa.X86_64.Sample.Rej4 (oJ half) +open VG.Proof.MlKem.X86_64 (Keep ea_at wp_countdown add_ofNat_zero pR WP.keep ofNat64_pred) +open VG.Proof.MlKem.X86_64.S4 +open VG.Proof.MlDsa.Sample (rnFold rnStep rnFold_snoc rnFold_length_le Stored stored_nil stored_frame G_eq G_length) +open VG.Proof.MlDsa.X86_64.Sample (rnBody_ok CoeffsWr) +open VG.Spec.MlKem (poly4 seed4) +open VG.Spec.MlDsa (Zq) +open VG.Proof.MlKem (xofByte xof_squeezeFrom_getElem) + +/-- ML-KEM's `S4.Env`, which this function shares. -/ +abbrev EnvK := VG.Proof.MlKem.X86_64.S4.Env + +/-- The 1008 bytes of output of seed `k`, which `vg_mldsa_rej_ntt_poly` samples from. -/ +abbrev Xb (σ : State) (k : Nat) : List Byte := Spec.MlDsa.G (B σ k) 1008 + +theorem Xb_getD (σ : State) (k : Nat) {p : Nat} (hp : p < 1008) : (Xb σ k).getD p 0 = xofByte (B σ k) p := by + have e := xof_squeezeFrom_getElem (B σ k) (pos := 0) (d := 1008) hp + rw [Nat.zero_add] at e + rw [List.getD_eq_getElem?_getD, List.getElem?_eq_getElem (by rw [Xb, G_length]; exact hp), Option.getD_some, ← e] + simp only [Xb, G_eq] + +/-- The coefficients of seed `k` after `n` iterations. -/ +abbrev Lt (σ : State) (k n : Nat) : List Zq := rnFold [] ((Xb σ k).take (3 * n)) + +theorem Lt_length_le (σ : State) (k n : Nat) : (Lt σ k n).length ≤ 256 := rnFold_length_le (by simp) _ + +theorem sx_ofNat {n : Nat} (h : n < 2 ^ 31) : BitVec.signExtend 64 (BitVec.ofNat 32 n) = BitVec.ofNat 64 n := by + have hm : (BitVec.ofNat 32 n).msb = false := by + rw [BitVec.msb_eq_decide]; simp only [BitVec.toNat_ofNat, decide_eq_false_iff_not]; omega + rw [BitVec.signExtend_eq_setWidth_of_msb_false hm] + apply BitVec.eq_of_toNat_eq + rw [BitVec.toNat_setWidth, BitVec.toNat_ofNat, BitVec.toNat_ofNat] + omega + +/-- A byte the frame's regions are apart from is unchanged. -/ +theorem frame_byte {rs : List Region} {m m' : Mem} (hf : Frame rs m m') {R : Region} (hd : ∀ r ∈ rs, R.Disjoint r) + {x : Addr} (hx : R.Contains x 1) : m' x = m x := + hf x fun r hr hc => hd r hr x hx hc + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +/-- Polynomial `K` lies in `a`. -/ +theorem sub_poly {K : Nat} (hK : K < 4) : Region.Sub (pR (poly4 (aP σ) K)) (aR σ) := Offset.sub_base _ (by omega) + +/-- A part of the scratch space is apart from polynomial `K`. -/ +theorem scr_poly {K : Nat} (hK : K < 4) {a n : Nat} (h : a + n ≤ 8192) : + ∀ r ∈ [pR (poly4 (aP σ) K)], Region.Disjoint ⟨at' σ a, n⟩ r := by + intro r hr + rw [List.mem_singleton.mp hr] + exact Region.Disjoint.sub_right (Region.Disjoint.sub_left hp.a_scr.symm (sub_scr h)) (sub_poly hK) + +omit hp in +/-- Two polynomials are apart. -/ +theorem poly_poly {K k : Nat} (hK : K < 4) (hk : k < 4) (hne : k ≠ K) : + ∀ r ∈ [pR (poly4 (aP σ) K)], (VG.Proof.MlDsa.Sample.polyR (poly4 (aP σ) k)).Disjoint r := by + intro r hr + simp only [List.mem_singleton] at hr; subst hr + exact Offset.disjoint (aP σ) (d := 1024 * k) (n := 1024) (e := 1024 * K) (k := 1024) (by omega) (by omega) + (by omega) + +/-- Writes to polynomial `K` keep `Env`. -/ +theorem env_poly {K : Nat} (hK : K < 4) {s s' : State} (he : EnvK σ s) + (hf : Frame [pR (poly4 (aP σ) K)] s.mem s'.mem) (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) + (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15], s'.gpr r = s.gpr r) : EnvK σ s' := by + have hsub := sub_poly (σ := σ) hK + refine ⟨hrd.trans he.rd, hwr.trans he.wr, by rw [hg .rbx (by simp), he.rbx], by rw [hg .r12 (by simp), he.r12], + by rw [hg .r13 (by simp), he.r13], by rw [hg .rsp (by simp), he.rsp], by rw [hg .r15 (by simp), he.r15], + fun i hi => ?_, he.frame.trans (hf.sub fun r hr => ?_)⟩ + · rw [hf.readW (Region.contains_self _ _) (scr_poly hp hK (by simp only [oSave]; omega)) (by decide)] + exact he.saved i hi + · simp only [List.mem_singleton] at hr; subst hr; exact ⟨aR σ, by simp, hsub⟩ + +/-- Each coefficient of polynomial `K` is writable. -/ +theorem coeffsWr {K : Nat} (hK : K < 4) {s : State} (he : EnvK σ s) : CoeffsWr s.wr (poly4 (aP σ) K) := by + intro j hj + rw [he.wr, hp.wr] + refine ⟨aR σ, by simp, ?_⟩ + rw [VG.Proof.MlDsa.Sample.coeffAddr, poly4, Offset.add_add] + exact Offset.contains_base _ (by omega) (by omega) + +end + +/-! ## A half -/ + +/-- At iteration `t` of half `h` of seed `K`, from `s₀`. -/ +structure HAt (σ s₀ : State) (K h t : Nat) (s : State) : Prop where + env : EnvK σ s + rsi : s.gpr .rsi = at' σ (oBuf + 504 * K) + BitVec.ofNat 64 (3 * t) + rdi : s.gpr .rdi = BitVec.ofNat 64 (Lt σ K (168 * h + t)).length + rcx : s.gpr .rcx = BitVec.ofNat 64 (168 - t) + rbp : s.gpr .rbp = poly4 (aP σ) K + out : ∀ p < 504, s.mem (at' σ (oBuf + 504 * K + p)) = xofByte (B σ K) (504 * h + p) + stored : Stored s.mem (poly4 (aP σ) K) (Lt σ K (168 * h + t)) + fr : Frame [pR (poly4 (aP σ) K)] s₀.mem s.mem + kp : Keep [.rax, .rdx, .r8, .rdi, .rsi, .rcx, .rbp] s₀ s + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +theorem hat_byte {s₀ : State} {K h t : Nat} (hh : h < 2) {s : State} (hI : HAt σ s₀ K h t s) {j : Nat} + (hj : 3 * t + j < 504) : + s.mem (s.gpr .rsi + BitVec.ofNat 64 j) = (Xb σ K).getD (3 * (168 * h + t) + j) 0 := by + have e := hI.out (3 * t + j) hj + rw [at'] at e + rw [hI.rsi, at', Offset.add_add, Offset.add_add, e, Xb_getD σ K (by omega)] + congr 1; omega + +theorem hat_regions {s₀ : State} {K h t : Nat} (hK : K < 4) {s : State} (hI : HAt σ s₀ K h t s) {j : Nat} + (hj : 3 * t + j < 504) : + InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 j) 1 := by + rw [hI.rsi, at', Offset.add_add, Offset.add_add] + exact in_scr' hp hI.env.rd hI.env.wr (by simp only [oBuf]; omega) + +/-- An iteration. -/ +theorem hat_step {s₀ : State} {K h t : Nat} (hK : K < 4) (hh : h < 2) (ht : t < 168) {s : State} + (hI : HAt σ s₀ K h t s) : + WP isa rnBody s fun s' => HAt σ s₀ K h (t + 1) s' ∧ s'.zf = some (BitVec.ofNat 64 (168 - t) - 1 == 0) := by + refine WP.mono (rnBody_ok s (aP := poly4 (aP σ) K) hI.rbp hI.rdi (Lt_length_le σ K _) (coeffsWr hp hK hI.env) + hI.stored (by simpa using hat_regions hp hK hI (j := 0) (by omega)) (hat_regions hp hK hI (by omega)) + (hat_regions hp hK hI (by omega))) fun s' ⟨hdi, hst, hf, hsi, hcx, hz, hk⟩ => ?_ + have e0 := hat_byte hh hI (j := 0) (by omega) + rw [add_ofNat_zero, Nat.add_zero] at e0 + have ht3 : Lt σ K (168 * h + (t + 1)) = rnStep (Lt σ K (168 * h + t)) ((Xb σ K).getD (3 * (168 * h + t)) 0) + ((Xb σ K).getD (3 * (168 * h + t) + 1) 0) ((Xb σ K).getD (3 * (168 * h + t) + 2) 0) := by + simp only [Lt] + rw [show 3 * (168 * h + (t + 1)) = 3 * (168 * h + t) + 3 by omega, + VG.Proof.MlDsa.X86_64.Sample.RejNtt.take_add_three _ (by rw [Xb, G_length]; omega), + rnFold_snoc _ (by rw [List.length_take, Xb, G_length]; omega)] + rw [e0, hat_byte hh hI (j := 1) (by omega), hat_byte hh hI (j := 2) (by omega), ← ht3] at hdi hst + have hk' : Keep [.rax, .rdx, .r8, .rdi, .rsi, .rcx] s s' := hk + refine ⟨⟨env_poly hp hK hI.env hf hk'.2.1 hk'.2.2 fun r hr => hk'.gpr (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr; rcases hr with rfl | rfl | rfl | rfl | rfl <;> decide), + by rw [hsi, hI.rsi, Offset.add_add, show 3 * t + 3 = 3 * (t + 1) by omega], hdi, + by rw [hcx, hI.rcx, ofNat64_pred (by omega) (by omega)]; rfl, + by rw [hk'.gpr (by decide), hI.rbp], + fun p hp' => by + rw [frame_byte hf (scr_poly hp hK (a := oBuf + 504 * K + p) (n := 1) (by simp only [oBuf]; omega)) + (Region.contains_self _ _)] + exact hI.out p hp', + hst, hI.fr.trans hf, (hI.kp.trans hk').mono (by simp)⟩, by rw [hz, hI.rcx]⟩ + +/-- Before a half: what holds of seed `K`. -/ +structure HPre (σ : State) (K h : Nat) (s : State) : Prop where + env : EnvK σ s + out : ∀ p < 504, s.mem (at' σ (oBuf + 504 * K + p)) = xofByte (B σ K) (504 * h + p) + j : s.mem.readW (at' σ (oJ + 8 * K)) 64 = BitVec.ofNat 64 (Lt σ K (168 * h)).length + stored : Stored s.mem (poly4 (aP σ) K) (Lt σ K (168 * h)) + +omit hp in +theorem sxB {K : Nat} (hK : K < 4) : BitVec.signExtend 64 (BitVec.ofNat 32 (oBuf + 504 * K)) = + BitVec.ofNat 64 (oBuf + 504 * K) := sx_ofNat (by simp only [oBuf]; omega) + +omit hp in +theorem sxP {K : Nat} (hK : K < 4) : BitVec.signExtend 64 (BitVec.ofNat 32 (1024 * K)) = + BitVec.ofNat 64 (1024 * K) := sx_ofNat (by omega) + +/-- The setup of a half: at iteration 0, from `j` kept. -/ +theorem hsetup_ok {K h : Nat} (hK : K < 4) {s : State} (hI : HPre σ K h s) : + WP isa (.block [.mov .rsi (.reg .rbx), .alu .add .rsi (.imm (BitVec.ofNat 32 (oBuf + 504 * K))), + .mov .rbp (.reg .r13), .alu .add .rbp (.imm (BitVec.ofNat 32 (1024 * K))), + .mov .rdi (.mem (at_ .rbx (oJ + 8 * K))), .mov32 .rcx (.imm 168)]) s (HAt σ s K h 0) := by + have hin : InRegions (s.rd ++ s.wr) (scr σ + BitVec.ofNat 64 (oJ + 8 * K)) 8 := + in_scr' hp hI.env.rd hI.env.wr (by simp only [oJ]; omega) + refine WP.mono (WP.keep [.rsi, .rbp, .rdi, .rcx] (Q := fun s' => s'.mem = s.mem ∧ + s'.gpr .rsi = at' σ (oBuf + 504 * K) ∧ s'.gpr .rbp = poly4 (aP σ) K ∧ + s'.gpr .rdi = BitVec.ofNat 64 (Lt σ K (168 * h)).length ∧ s'.gpr .rcx = BitVec.ofNat 64 168) + (by + xrun [hI.env.rbx, hI.env.r13, sxB hK, sxP hK, hin, hI.j] + rfl) + rfl) fun s₁ ⟨⟨hm, hsi, hbp, hdi, hcx⟩, k₁⟩ => ?_ + exact ⟨hI.env.keep hm k₁ (by decide), by rw [hsi, add_ofNat_zero], + by rw [hdi, Nat.add_zero], by rw [hcx], hbp, by rw [hm]; exact hI.out, by rw [hm, Nat.add_zero]; exact hI.stored, + by rw [hm]; exact Frame.refl _ _, k₁.mono (by simp)⟩ + +/-- The loop of a half, from iteration 0. -/ +theorem hloop_ok {s₀ : State} {K h : Nat} (hK : K < 4) (hh : h < 2) {s : State} (hI : HAt σ s₀ K h 0 s) : + WP isa (.loop rnBody .ne) s (HAt σ s₀ K h 168) := by + refine wp_countdown (N := 168) (by decide) (by decide) (fun t u => HAt σ s₀ K h t u) + (fun t ht u hu _ => WP.mono (hat_step hp hK hh ht hu) fun u' ⟨hu', hz⟩ => ⟨hu', ?_, by rw [hz, hu.rcx]⟩) + (fun _ h => h) hI hI.rcx + rw [hu'.rcx, hu.rcx, ofNat64_pred (by omega) (by omega)]; rfl + +/-- A half of seed `K`: the coefficients of the first `504 (h + 1)` bytes, +writing only polynomial `K`. -/ +theorem half_ok {K h : Nat} (hK : K < 4) (hh : h < 2) {s : State} (hI : HPre σ K h s) : + WP isa (half K) s (HAt σ s K h 168) := + WP.seq (WP.mono (hsetup_ok hp hK hI) fun _ h₁ => hloop_ok hp hK hh h₁) + +end + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Scalar.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Scalar.lean new file mode 100644 index 000000000..acc7780d0 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Scalar.lean @@ -0,0 +1,264 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4CT +import VerifiedGarbage.Proof.MlDsa.Arith.Mem + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4` + +Untrusted: everything here is checked by Lean. The baseline implementation +calls `vg_mldsa_rej_ntt_poly` on each seed, between the prologue and the +epilogue of the one for AVX2 (`Rej4Top.lean`): after the call on seed `K`, +polynomial `K` is the seed's `RejNTTPoly` if it has 256 coefficients, and +`r14` records whether the first `K + 1` do (`PC`), as in +`vg_mlkem_sample_ntt4` (`MlKem/X86_64/S4Scalar.lean`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Impl.MlDsa.X86_64.Sample.Rej4 (rejNTT4) +open VG.Proof.MlKem.X86_64 (Keep sample4K relInv taintRel relStart Rel2 pR ret_disj24 stk_disj24' + ce_bytesAt24 ce_gpr' WP.keep nosp_of) +open VG.Proof.MlKem.X86_64.S4 (R4 Pre aP at' sd scr aR scrR stkR sdR pre_of pub_scr pub_aP pub_B pub_sd pub_sp + env_rbx pro_ok I0 sub_scr sub_poly seed_bytes cov scr6144_lt sx34 sx1024 sx6144 c_sub c_disj and14_ok epi_eq + cRd cWr in_scr') +open VG.Proof.MlDsa.X86_64.Sample (rnK nil_regs rejNTT_correct rejNTT_ct) +open VG.Proof.MlDsa.Sample (rnFold toPoly) +open VG.Proof.MlDsa.Arith (polyIs_frame) +open VG.Spec.MlKem (poly4) +open VG.Spec.MlDsa (G PolyIs) + +theorem rn_nosp : NoSp Impl.MlDsa.X86_64.Sample.rejNTT := nosp_of (by decide +kernel) + +theorem rn_depth : Impl.MlDsa.X86_64.Sample.rejNTT.depth = 2 := by decide +kernel + +/-- Before the call on seed `K`. -/ +structure PC (σ : State) (K : Nat) (s : State) : Prop where + env : EnvK σ s + r14 : s.gpr .r14 = BitVec.ofNat 64 (okN σ K) + polys : ∀ k < K, (Lt σ k 336).length = 256 → PolyIs s.mem (poly4 (aP σ) k) (toPoly (Lt σ k 336)) + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +/-- `PC` after code that writes no memory and keeps its registers. -/ +theorem PC.keep {K : Nat} {s s' : State} (h : PC σ K s) (hm : s'.mem = s.mem) {rs : List Reg} + (hk : Keep rs s s') (hrs : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15, .r14], r ∉ rs) : PC σ K s' := + ⟨h.env.keep hm hk fun r hr => hrs r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr ⊢; rcases hr with h | h | h | h | h <;> simp [h]), + by rw [hk.gpr (hrs .r14 (by simp)), h.r14], by rw [hm]; exact h.polys⟩ + +/-- `PC` after the call. -/ +theorem PC.call {K : Nat} (hK : K < 4) {s s' : State} (h : PC σ K s) (hrd : s'.rd = s.rd) + (hwr : s'.wr = s.wr) (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15, .r14], s'.gpr r = s.gpr r) + (hf : Frame (cWr σ K ++ [stkR σ]) s.mem s'.mem) : PC σ K s' := by + refine ⟨⟨hrd.trans h.env.rd, hwr.trans h.env.wr, by rw [hg .rbx (by simp), h.env.rbx], + by rw [hg .r12 (by simp), h.env.r12], by rw [hg .r13 (by simp), h.env.r13], by rw [hg .rsp (by simp), h.env.rsp], + by rw [hg .r15 (by simp), h.env.r15], fun i hi => ?_, h.env.frame.trans (hf.sub (c_sub (σ := σ) hK))⟩, + by rw [hg .r14 (by simp), h.r14], fun k hk e => ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (c_disj hp hK (by simp only [oSave, oScalar]; omega)) (by decide)] + exact h.env.saved i hi + · refine polyIs_frame hf (fun r hr => ?_) (h.polys k hk e) + simp only [List.mem_append, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with (rfl | rfl) | rfl + · have hd := Offset.disjoint (aP σ) (d := 1024 * k) (n := 1024) (e := 1024 * K) (k := 1024) (by omega) (by omega) + (by omega) + simpa [poly4] using hd + · exact (hp.a_scr.sub_left (sub_poly (by omega))).sub_right (sub_scr (by simp only [oScalar]; omega)) + · exact (hp.stk_a.sub_right (sub_poly (by omega))).symm + +/-- The arguments of the call on seed `K`. -/ +structure ArgI (σ : State) (K : Nat) (s : State) : Prop where + pinv : PC σ K s + rdi : s.gpr .rdi = sd σ + BitVec.ofNat 64 (34 * K) + rsi : s.gpr .rsi = poly4 (aP σ) K + rdx : s.gpr .rdx = at' σ oScalar + +omit hp in +theorem argsK_ok {K : Nat} (hK : K < 4) {s : State} (h : PC σ K s) : + WP isa (.block [.mov .rdi (.reg .r12), .alu .add .rdi (.imm (BitVec.ofNat 32 (34 * K))), + .mov .rsi (.reg .r13), .alu .add .rsi (.imm (BitVec.ofNat 32 (1024 * K))), .mov .rdx (.reg .rbx), + .alu .add .rdx (.imm (BitVec.ofNat 32 oScalar))]) s (ArgI σ K) := + WP.mono (WP.keep [.rdi, .rsi, .rdx] (Q := fun s' => s'.mem = s.mem ∧ + s'.gpr .rdi = sd σ + BitVec.ofNat 64 (34 * K) ∧ s'.gpr .rsi = poly4 (aP σ) K ∧ s'.gpr .rdx = at' σ oScalar) + (by xrun [h.env.r12, h.env.r13, h.env.rbx, sx34 hK, sx1024 hK, sx6144]; exact ⟨rfl, rfl⟩) rfl) + fun _ ⟨⟨hm₂, hdi, hsi, hdx⟩, k₂⟩ => ⟨h.keep hm₂ k₂ (by decide), hdi, hsi, hdx⟩ + +theorem argK_kS {K : Nat} (hK : K < 4) {s : State} (h : ArgI σ K s) : + (below (s.gpr .rsp) 24).Disjoint ⟨sd σ + BitVec.ofNat 64 (34 * K), 34⟩ := by + rw [h.pinv.env.rsp]; exact hp.stk_sd.sub_right (Offset.sub_base (sd σ) (d := 34 * K) (n := 34) (by omega)) + +theorem argK_pre {K : Nat} (hK : K < 4) {s : State} (h : ArgI σ K s) : + rnK.pre (s.callEntry.withRegions (cRd σ K) (cWr σ K)) := by + have hsp : s.gpr .rsp = σ.gpr .rsp := h.pinv.env.rsp + have kS := argK_kS hp hK h + have kA : (below (s.gpr .rsp) 24).Disjoint (pR (poly4 (aP σ) K)) := by + rw [hsp]; exact hp.stk_a.sub_right (sub_poly (σ := σ) hK) + have kZ : (below (s.gpr .rsp) 24).Disjoint ⟨at' σ oScalar, 2048⟩ := by + rw [hsp]; exact hp.stk_scr.sub_right (sub_scr (σ := σ) (a := oScalar) (n := 2048) (by simp only [oScalar]; omega)) + simp only [rnK, State.withRegions_gpr, State.withRegions_rd, State.withRegions_wr, + ce_gpr' s (by decide : Reg.rdi ≠ .rsp), ce_gpr' s (by decide : Reg.rsi ≠ .rsp), + ce_gpr' s (by decide : Reg.rdx ≠ .rsp), h.rdi, h.rsi, h.rdx] + exact ⟨trivial, trivial, + (hp.sd_a.sub_left (Offset.sub_base _ (by omega))).sub_right (sub_poly hK), + (hp.sd_scr.sub_left (Offset.sub_base _ (by omega))).sub_right (sub_scr (by simp only [oScalar]; omega)), + (hp.a_scr.sub_left (sub_poly hK)).sub_right (sub_scr (by simp only [oScalar]; omega)), + ret_disj24 s kS, ret_disj24 s kA, ret_disj24 s kZ, stk_disj24' s kS, stk_disj24' s kA, stk_disj24' s kZ, + scr6144_lt hp⟩ + +/-- After the call on seed `K`. -/ +structure CallI (σ : State) (K : Nat) (s : State) : Prop where + pinv : PC σ K s + rax : (s.gpr .rax).setWidth 32 = if (Lt σ K 336).length = 256 then 1 else 0 + poly : (Lt σ K 336).length = 256 → PolyIs s.mem (poly4 (aP σ) K) (toPoly (Lt σ K 336)) + +theorem callK_ok {K : Nat} (hK : K < 4) {s : State} (h : ArgI σ K s) : + WP isa (.call "vg_mldsa_rej_ntt_poly" Impl.MlDsa.X86_64.Sample.rejNTT) s (CallI σ K) := by + have hcv := cov hp h.pinv.env hK + refine WP.call rejNTT_correct rn_nosp (by rw [rn_depth]; decide) (argK_pre hp hK h) hcv.1 hcv.2 + fun s₃ hrd hwr hcs hf _ ⟨s₃', hm₃, hg₃, hpost⟩ => ?_ + rw [rn_depth, h.pinv.env.rsp] at hf + have h₃ : PC σ K s₃ := h.pinv.call hp hK hrd hwr (fun r hr => hcs r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide)) hf + simp only [rnK, State.withRegions_gpr, State.withRegions_mem, ce_gpr' s (by decide : Reg.rdi ≠ .rsp), + ce_gpr' s (by decide : Reg.rsi ≠ .rsp), h.rdi, h.rsi, hm₃, ce_bytesAt24 s (n := 34) (by decide) (argK_kS hp hK h), + seed_bytes hp hK h.pinv.env.frame] at hpost + rw [hg₃ .rax (by decide)] at hpost + simp only [← Lt_336] at hpost + exact ⟨h₃, hpost.1, hpost.2⟩ + +omit hp in +theorem okN_succ' (K : Nat) : BitVec.setWidth 64 ((BitVec.ofNat 64 (okN σ K)).setWidth 32 &&& + (if (Lt σ K 336).length = 256 then 1 else 0)) = BitVec.ofNat 64 (okN σ (K + 1)) := by + simp only [okN, List.range_succ, List.all_append, List.all_cons, List.all_nil, Bool.and_true] + by_cases e : (Lt σ K 336).length = 256 + · rw [ite_eq_left e, show ((Lt σ K 336).length == 256) = true by simpa using e] + cases (List.range K).all fun k => (Lt σ k 336).length == 256 <;> rfl + · rw [ite_eq_right e, show ((Lt σ K 336).length == 256) = false by simpa using e] + cases (List.range K).all fun k => (Lt σ k 336).length == 256 <;> rfl + +omit hp in +theorem andK_ok {K : Nat} {s : State} (h : CallI σ K s) : + WP isa (.block [.alu32 .and .r14 (.reg .rax)]) s (PC σ (K + 1)) := by + refine WP.mono (and14_ok s) fun s₄ ⟨⟨h14, hm₄⟩, k₄⟩ => ?_ + refine ⟨h.pinv.env.keep hm₄ k₄ (by decide), by rw [h14, h.pinv.r14, h.rax, okN_succ'], fun k hk e => ?_⟩ + rw [hm₄] + by_cases ek : k = K + · subst ek; exact h.poly e + · exact h.pinv.polys k (by omega) e + +theorem callK_ok' {K : Nat} (hK : K < 4) {s : State} (h : PC σ K s) : WP isa (Impl.MlDsa.X86_64.Sample.Rej4.callK K) s (PC σ (K + 1)) := + WP.seq (WP.mono (argsK_ok hK h) fun _ h₂ => WP.seq (WP.mono (callK_ok hp hK h₂) fun _ h₃ => andK_ok h₃)) + +/-- The return value, and the callee-saved registers restored. -/ +theorem endS_ok {s : State} (h : PC σ 4 s) : + WP isa (.block epi) s fun s' => r4K.post σ s' ∧ gprPreserved σ s' := by + have hin : ∀ i < 5, InRegions (s.rd ++ s.wr) (scr σ + BitVec.ofNat 64 (oSave + 8 * i)) 8 := fun i hi => + in_scr' hp h.env.rd h.env.wr (by simp only [oSave]; omega) + rw [epi_eq] + refine WP.mono (WP.keep [.rax, .r14, .r13, .r12, .rbp, .rbx] (Q := fun s' => s'.mem = s.mem ∧ + (s'.gpr .rax).setWidth 32 = (s.gpr .r14).setWidth 32 ∧ + s'.gpr .r14 = s.mem.readW (at' σ (oSave + 8 * 4)) 64 ∧ s'.gpr .r13 = s.mem.readW (at' σ (oSave + 8 * 3)) 64 ∧ + s'.gpr .r12 = s.mem.readW (at' σ (oSave + 8 * 2)) 64 ∧ s'.gpr .rbp = s.mem.readW (at' σ (oSave + 8 * 1)) 64 ∧ + s'.gpr .rbx = s.mem.readW (at' σ (oSave + 8 * 0)) 64) + (by + have h0 := hin 0 (by decide); have h1 := hin 1 (by decide); have h2 := hin 2 (by decide) + have h3 := hin 3 (by decide); have h4 := hin 4 (by decide) + simp only [oSave, Nat.reduceMul, Nat.reduceAdd] at h0 h1 h2 h3 h4 + xrun [h0, h1, h2, h3, h4, h.env.rbx] + exact ⟨rfl, rfl, rfl, rfl, rfl⟩) + (by decide)) fun s' ⟨⟨hm, hax, h14, h13, h12, hbp, hbx⟩, k⟩ => ?_ + refine ⟨⟨?_, fun k hk e => ?_⟩, fun r hr => ?_, ?_⟩ + · rw [hax, h.r14, okN] + simp only [Lt_336] + split <;> rfl + · rw [hm, ← Lt_336] + rw [← Lt_336] at e + exact h.polys k hk e + · simp only [calleeSaved, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl | rfl + · rw [hbx]; exact h.env.saved 0 (by decide) + · rw [hbp]; exact h.env.saved 1 (by decide) + · rw [k.gpr (by decide)]; exact h.env.rsp + · rw [h12]; exact h.env.saved 2 (by decide) + · rw [h13]; exact h.env.saved 3 (by decide) + · rw [h14]; exact h.env.saved 4 (by decide) + · rw [k.gpr (by decide)]; exact h.env.r15 + · rw [hm] + exact h.env.frame.readW (Region.contains_self _ _) (by + simpa using ⟨hp.ret_a, hp.ret_scr, Offset.base_disjoint_below (σ.gpr .rsp) (n := 24) (k := 8) (by omega)⟩) + (by decide) + +theorem scalar_body_ok {s : State} (h : I0 σ s) : + WP isa (.seq (Impl.MlDsa.X86_64.Sample.Rej4.callK 0) (.seq (Impl.MlDsa.X86_64.Sample.Rej4.callK 1) + (.seq (Impl.MlDsa.X86_64.Sample.Rej4.callK 2) (.seq (Impl.MlDsa.X86_64.Sample.Rej4.callK 3) (.block epi))))) + s fun s' => + r4K.post σ s' ∧ gprPreserved σ s' := by + have p₀ : PC σ 0 s := ⟨h.env, by rw [h.r14]; rfl, fun _ h _ => absurd h (by omega)⟩ + exact WP.seq (WP.mono (callK_ok' hp (by decide) p₀) fun _ p₁ => WP.seq (WP.mono (callK_ok' hp (by decide) p₁) + fun _ p₂ => WP.seq (WP.mono (callK_ok' hp (by decide) p₂) fun _ p₃ => + WP.seq (WP.mono (callK_ok' hp (by decide) p₃) fun _ p₄ => endS_ok hp p₄)))) + +end + +theorem correct_scalar (σ : State) (hs : r4K.pre σ) : + ∃ t s', Exec isa rejNTT4 σ t s' ∧ abiPreserved σ s' ∧ r4K.post σ s' := by + have hp := pre_of hs + obtain ⟨t, s', he, hF⟩ := WP.seq (WP.mono (pro_ok hp) fun _ h => scalar_body_ok hp h) + exact ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he hF.2, hF.1⟩ + +/-! ## Constant time -/ + +/-- The call on seed `K`. -/ +theorem call_ct {K : Nat} (hK : K < 4) : + RelCT isa (R4 fun σ s => ArgI σ K s) (.call "vg_mldsa_rej_ntt_poly" Impl.MlDsa.X86_64.Sample.rejNTT) + (R4 fun σ s => CallI σ K s) := + relInv (fun σ s hp h => callK_ok (pre_of hp) hK h) (RelCT.callEx rejNTT_correct rejNTT_ct + fun s₁ s₂ ⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩ => by + have hsp : s₁.gpr .rsp = s₂.gpr .rsp := by rw [h₁.pinv.env.rsp, h₂.pinv.env.rsp, pub_sp hq] + refine ⟨_, _, _, _, argK_pre (pre_of p₁) hK h₁, argK_pre (pre_of p₂) hK h₂, ?_, + (cov (pre_of p₁) h₁.pinv.env hK).1, (cov (pre_of p₁) h₁.pinv.env hK).2, + (cov (pre_of p₂) h₂.pinv.env hK).1, (cov (pre_of p₂) h₂.pinv.env hK).2, hsp⟩ + simp only [rnK, State.withRegions_gpr, State.withRegions_mem, State.callEntry_rsp, + ce_gpr' _ (by decide : Reg.rdi ≠ .rsp), ce_gpr' _ (by decide : Reg.rsi ≠ .rsp), + ce_gpr' _ (by decide : Reg.rdx ≠ .rsp), h₁.rdi, h₂.rdi, h₁.rsi, h₂.rsi, h₁.rdx, h₂.rdx] + rw [ce_bytesAt24 s₁ (n := 34) (by decide) (argK_kS (pre_of p₁) hK h₁), + ce_bytesAt24 s₂ (n := 34) (by decide) (argK_kS (pre_of p₂) hK h₂), + seed_bytes (pre_of p₁) hK h₁.pinv.env.frame, seed_bytes (pre_of p₂) hK h₂.pinv.env.frame, pub_B hq hK] + simp only [pub_sd hq, pub_aP hq, at', pub_scr hq, hsp, and_self]) + +/-- The call on seed `K`, given the taint analysis of its arguments. -/ +theorem callK_ct {K : Nat} (hK : K < 4) {hc : VG.Taint.Hint X86_64.Taint.T} + (c : (taint.check (X86_64.Taint.ofRegs [.r12, .r13, .rbx]) + (.block [.mov .rdi (.reg .r12), .alu .add .rdi (.imm (BitVec.ofNat 32 (34 * K))), + .mov .rsi (.reg .r13), .alu .add .rsi (.imm (BitVec.ofNat 32 (1024 * K))), .mov .rdx (.reg .rbx), + .alu .add .rdx (.imm (BitVec.ofNat 32 oScalar))]) hc).isSome = true) : + RelCT isa (R4 fun σ s => PC σ K s) (Impl.MlDsa.X86_64.Sample.Rej4.callK K) (R4 fun σ s => PC σ (K + 1) s) := + RelCT.seq (relInv (fun σ s _ h => argsK_ok hK h) (taintRel [.r12, .r13, .rbx] + (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl + · rw [h₁.env.r12, h₂.env.r12, pub_sd hq] + · rw [h₁.env.r13, h₂.env.r13, pub_aP hq] + · exact env_rbx hq h₁.env h₂.env) c)) + (RelCT.seq (call_ct hK) (relInv (fun σ s _ h => andK_ok h) (taintRel [] nil_regs (by taint_decide)))) + +theorem ct_scalar : ConstantTime isa r4K.pre r4K.pub rejNTT4 := by + refine relStart (Q := fun _ _ => True) (RelCT.seq (RelCT.mono (relInv (I' := fun σ s => PC σ 0 s) + (fun σ s hp h => by + subst h + exact WP.mono (pro_ok (pre_of hp)) fun _ h => ⟨h.env, by rw [h.r14]; rfl, fun _ h _ => absurd h (by omega)⟩) + (taintRel [.rdi, .rsi, .rdx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + subst h₁ h₂ + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl + exacts [hq.1, hq.2.1, hq.2.2.1]) (by taint_decide))) (fun _ _ h => h) fun _ _ h => h) ?_) + refine RelCT.seq (callK_ct (K := 0) (by decide) (by taint_decide)) (RelCT.seq (callK_ct (K := 1) (by decide) + (by taint_decide)) (RelCT.seq (callK_ct (K := 2) (by decide) (by taint_decide)) + (RelCT.seq (callK_ct (K := 3) (by decide) (by taint_decide)) ?_))) + exact taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) (by taint_decide) + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Sq.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Sq.lean new file mode 100644 index 000000000..e5abcc586 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Sq.lean @@ -0,0 +1,156 @@ +import VerifiedGarbage.Proof.MlKem.X86_64.S4Squeeze +import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejNtt4 + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4_avx2`, squeezing + +Untrusted: everything here is checked by Lean. `vg_mldsa_rej_ntt_poly4_avx2` +absorbs the seeds and squeezes three blocks of each as +`vg_mlkem_sample_ntt4_avx2` does (`Proof/MlKem/X86_64/S4*.lean`, whose +precondition, layout and invariants it shares), then three more to the same +buffers. `SqT σ t n` is `SqInv σ n` (`S4Squeeze.lean`) after `t` blocks +squeezed before: the states are permuted `t + n` times, and the buffers hold +bytes `168 t` to `168 (t + n)` of each seed's output. A squeeze writes only +below the saved registers (`sqT_ok`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Proof.MlKem.X86_64 (Keep ea_at) +open VG.Proof.MlKem.X86_64.S4 +open VG.Spec.MlKem (seed4 poly4) +open VG.Spec.Sha3 (keccakF RC) +open VG.Proof.Sha3 (byteOf iterF iterF_succ iterF_keccakF) +open VG.Proof.MlKem (xofByte) +open VG.Proof.Sha3.X86_64.X4 (la Lanes4 byte_of_lanes4 permute4_ok) + +/-- After `n` squeezes, `t` blocks after the absorption. -/ +structure SqT (σ : State) (t n : Nat) (s : State) : Prop where + env : Env σ s + r14 : s.gpr .r14 = 1 + rc : ∀ r < 24, ∀ k < 4, s.mem.readW (la (scr σ) (50 + r) k) 64 = RC r + lanes : Lanes4 s.mem (scr σ) fun k => iterF (t + n) (A0 (B σ k)) + buf : ∀ k < 4, ∀ p < 168 * n, s.mem (at' σ (oBuf + 504 * k + p)) = xofByte (B σ k) (168 * t + p) + +theorem sqT_of {σ s : State} (h : SqInv σ 0 s) : SqT σ 0 0 s := + ⟨h.env, h.r14, h.rc, h.lanes, fun _ _ p hp => absurd hp (by omega)⟩ + +/-- The low part of the scratch space, which the squeezes write. -/ +abbrev lowR (σ : State) : Region := ⟨scr σ, oSave⟩ + +/-- The permutation, after `n` squeezes. -/ +theorem permT_ok {σ : State} (hp : Pre σ) {t n : Nat} (hn : n < 3) {s : State} (h : SqT σ t n s) + {rest : Prog isa} {Q : State → Prop} (kont : ∀ s', Env σ s' ∧ s'.gpr .r14 = 1 ∧ + (∀ r < 24, ∀ k < 4, s'.mem.readW (la (scr σ) (50 + r) k) 64 = RC r) ∧ + Lanes4 s'.mem (scr σ) (fun k => iterF (t + n + 1) (A0 (B σ k))) ∧ + (∀ k < 4, ∀ p < 168 * n, s'.mem (at' σ (oBuf + 504 * k + p)) = xofByte (B σ k) (168 * t + p)) ∧ + Frame [lowR σ] s.mem s'.mem → WP isa rest s' Q) : + WP isa (.seq (.block permArgs) (.seq Impl.Sha3.X86_64.X4.permute4 rest)) s Q := by + refine WP.seq (WP.mono (args_ok h.env) fun s₁ ⟨⟨hm, hdi, hsi, hdx, hcx⟩, k₁⟩ => ?_) + have hrd : s₁.rd = σ.rd := k₁.2.1.trans h.env.rd + have hwr : s₁.wr = σ.wr := k₁.2.2.trans h.env.wr + refine WP.seq (WP.mono (permute4_ok (A := fun k => iterF (t + n) (A0 (B σ k))) + (pre4 hp hrd hwr (by rw [hm]; exact h.rc)) hdi hsi hdx (by rw [hcx, at', at', Offset.add_add]) + (by rw [hm]; exact h.lanes)) fun s₂ ⟨hl, hf, hrd₂, hwr₂, _, hg⟩ => kont s₂ ?_) + have hsub : ∀ r ∈ [(⟨scr σ, 800⟩ : Region), ⟨at' σ 800, 800⟩], Region.Sub r ⟨scr σ, oSave⟩ := by + intro r hr + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl + · exact Region.sub_prefix (by simp only [oSave]; omega) + · exact Offset.sub_base _ (by simp only [oSave]; omega) + refine ⟨Env.low (h.env.keep hm k₁ (by decide)) hsub hf hrd₂ hwr₂ fun r hr => hg r + (by simp only [List.mem_cons, List.not_mem_nil, or_false] at hr; rcases hr with rfl | rfl | rfl | rfl | rfl <;> decide) + (by simp only [List.mem_cons, List.not_mem_nil, or_false] at hr; rcases hr with rfl | rfl | rfl | rfl | rfl <;> decide), + by rw [hg _ (by decide) (by decide), k₁.gpr (by decide), h.r14], fun r hr k hk => ?_, ?_, fun k hk p hp' => ?_, + by rw [← hm]; exact hf.sub fun r hr => ⟨lowR σ, List.mem_singleton_self _, hsub r hr⟩⟩ + · rw [hf.readW (Region.contains_self _ _) (by + simpa using ⟨Offset.disjoint_base (scr σ) (k := 800) (d := 32 * (50 + r) + 8 * k) (n := 8) (by omega) (by omega), + Offset.disjoint (scr σ) (d := 32 * (50 + r) + 8 * k) (n := 8) (e := 800) (k := 800) (by omega) (by omega) + (by omega)⟩) (by decide), hm] + exact h.rc r hr k hk + · intro i hi k hk + rw [hl i hi k hk] + rfl + · rw [buf_frame (by simpa using ⟨Offset.disjoint_base (scr σ) (k := 800) (d := oBuf) (n := 2016) + (by simp only [oBuf]; omega) (by simp only [oBuf]; omega), Offset.disjoint (scr σ) (d := oBuf) (n := 2016) + (e := 800) (k := 800) (by simp only [oBuf]; omega) (by simp only [oBuf]; omega) (by omega)⟩) hf hk (by omega), hm] + exact h.buf k hk p hp' + +/-- During the copy of block `n`: the first `I` lanes of state `K` copied, +and all of the states before it. -/ +structure EXT (σ : State) (m₀ : Mem) (t n K I : Nat) (s : State) : Prop where + env : Env σ s + r14 : s.gpr .r14 = 1 + rc : ∀ r < 24, ∀ k < 4, s.mem.readW (la (scr σ) (50 + r) k) 64 = RC r + lanes : Lanes4 s.mem (scr σ) (fun k => iterF (t + n + 1) (A0 (B σ k))) + buf : ∀ k < 4, ∀ p < 504, (p < 168 * n ∨ (168 * n ≤ p ∧ p < 168 * n + 168 ∧ (k < K ∨ (k = K ∧ p < 168 * n + 8 * I)))) → + s.mem (at' σ (oBuf + 504 * k + p)) = xofByte (B σ k) (168 * t + p) + fr : Frame [lowR σ] m₀ s.mem + +open VG.Proof.Sha3.X86_64 (wp_movm wp_store wp_nil) in +theorem extT_step {σ : State} (hp : Pre σ) {m₀ : Mem} {t n K I : Nat} (hn : n < 3) (hK : K < 4) (hI : I < 21) + {s : State} (h : EXT σ m₀ t n K I s) : + WP isa (.block [.mov .rax (.mem (at_ .rbx (32 * I + 8 * K))), + .store (at_ .rbx (oBuf + 504 * K + 168 * n + 8 * I)) .rax]) s (EXT σ m₀ t n K (I + 1)) := by + refine wp_movm (a := at' σ (32 * I + 8 * K)) (by rw [ea_at, h.env.rbx]) + (in_scr' hp h.env.rd h.env.wr (by omega)) fun s₁ u₁ => wp_store (a := at' σ (oBuf + 504 * K + 168 * n + 8 * I)) + (by rw [ea_at, u₁.other _ (by decide), h.env.rbx]) (by rw [u₁.wr]; exact in_scr hp h.env.wr (by simp only [oBuf]; omega)) + fun s₂ g₂ m₂ r₂ w₂ => wp_nil ?_ + have hm : s₂.mem = s.mem.writeW (at' σ (oBuf + 504 * K + 168 * n + 8 * I)) (s.mem.readW (at' σ (32 * I + 8 * K)) 64) := by + rw [m₂, u₁.mem, u₁.gpr] + have hsub : Region.Sub ⟨at' σ (oBuf + 504 * K + 168 * n + 8 * I), 8⟩ (lowR σ) := + Offset.sub_base _ (by simp only [oBuf, oSave]; omega) + have hf : Frame [⟨at' σ (oBuf + 504 * K + 168 * n + 8 * I), 8⟩] s.mem s₂.mem := by + rw [hm]; exact (Frame.refl _ _).writeW (List.mem_singleton_self _) _ (Region.contains_self _ _) + refine ⟨Env.low h.env (fun r hr => by simp only [List.mem_singleton] at hr; subst hr; exact hsub) hf + (r₂.trans u₁.rd) (w₂.trans u₁.wr) + fun r hr => by + rw [g₂, u₁.other r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl <;> decide)], + by rw [g₂, u₁.other _ (by decide), h.r14], fun r hr k hk => ?_, fun i hi k hk => ?_, fun k hk p hp' hc => ?_, + h.fr.trans (hf.sub fun r hr => ⟨lowR σ, List.mem_singleton_self _, by + simp only [List.mem_singleton] at hr; subst hr; exact hsub⟩)⟩ + · have e := readW_writeW_off s.mem (scr σ) (s.mem.readW (at' σ (32 * I + 8 * K)) 64) (d := 32 * (50 + r) + 8 * k) + (e := oBuf + 504 * K + 168 * n + 8 * I) (n := 8) (by omega) (by simp only [oBuf]; omega) + (by simp only [oBuf]; omega) + rw [hm]; exact e.trans (h.rc r hr k hk) + · have e := readW_writeW_off s.mem (scr σ) (s.mem.readW (at' σ (32 * I + 8 * K)) 64) (d := 32 * i + 8 * k) + (e := oBuf + 504 * K + 168 * n + 8 * I) (n := 8) (by omega) (by simp only [oBuf]; omega) + (by simp only [oBuf]; omega) + rw [hm]; exact e.trans (h.lanes i hi k hk) + · rw [hm] + by_cases hw : k = K ∧ 168 * n + 8 * I ≤ p ∧ p < 168 * n + 8 * I + 8 + · obtain ⟨rfl, h₁, h₂⟩ := hw + rw [wb_in _ _ _ (by omega) (by simp only [oBuf]; omega) (by decide), + show 8 * (oBuf + 504 * k + p - (oBuf + 504 * k + 168 * n + 8 * I)) = 8 * (p - 168 * n - 8 * I) by omega, + byte_readW _ _ (by omega), at', Offset.add_add, + show 32 * I + 8 * k + (p - 168 * n - 8 * I) = 32 * ((8 * I + (p - 168 * n - 8 * I)) / 8) + 8 * k + + (8 * I + (p - 168 * n - 8 * I)) % 8 by omega, + byte_of_lanes4 h.lanes hk (by omega), show t + n + 1 = (t + n) + 1 from rfl, + ← xofByte_A0 (B_length σ k) (by omega), + show 168 * (t + n) + (8 * I + (p - 168 * n - 8 * I)) = 168 * t + p by omega] + · rw [wb_out _ _ _ (by simp only [oBuf]; omega) (by simp only [oBuf]; omega) (by simp only [oBuf]; omega)] + exact h.buf k hk p hp' (by omega) + +/-- Block `n` of each state's output. -/ +theorem extractT_ok {σ : State} (hp : Pre σ) {m₀ : Mem} {t n : Nat} (hn : n < 3) {s : State} + (h : EXT σ m₀ t n 0 0 s) : + WP isa (.block (extract n)) s fun s' => SqT σ t (n + 1) s' ∧ Frame [lowR σ] m₀ s'.mem := by + rw [extract_eq] + refine WP.mono (wp_range_flatMap (M := isa) (fun K s => EXT σ m₀ t n K 0 s) (fun K s hK h => ?_) 4 (Nat.le_refl _) + s h) fun s' h' => ⟨⟨h'.env, h'.r14, h'.rc, h'.lanes, fun k hk p hp' => h'.buf k hk p (by omega) (by omega)⟩, h'.fr⟩ + refine WP.mono (wp_range_flatMap (M := isa) (fun I s => EXT σ m₀ t n K I s) (fun I s hI h => extT_step hp hn hK hI h) + 21 (Nat.le_refl _) s h) fun s' h' => + ⟨h'.env, h'.r14, h'.rc, h'.lanes, fun k hk p hp' hc => h'.buf k hk p hp' (by omega), h'.fr⟩ + +/-- `squeeze4 n`: after `n + 1` squeezes; it writes only below the saved +registers. -/ +theorem sqT_ok {σ : State} (hp : Pre σ) {t n : Nat} (hn : n < 3) {s : State} (h : SqT σ t n s) : + WP isa (squeeze4 n) s fun s' => SqT σ t (n + 1) s' ∧ Frame [lowR σ] s.mem s'.mem := by + unfold squeeze4 + exact permT_ok hp hn h fun s' ⟨he, h14, hrc, hl, hb, hf⟩ => + extractT_ok hp hn ⟨he, h14, hrc, hl, fun k hk p _ hc => hb k hk p (by omega), hf⟩ + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Top.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Top.lean new file mode 100644 index 000000000..282e42ee0 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Top.lean @@ -0,0 +1,412 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Parse +import VerifiedGarbage.Proof.MlKem.X86_64.S4Top + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4_avx2`, correctness + +Untrusted: everything here is checked by Lean. The pieces, in order: the +prologue, the round constants and the padded seeds, and three squeezes (as +in `vg_mlkem_sample_ntt4_avx2`); the counts zeroed; the first half of each +seed (`P1`); three more squeezes; the second half of each seed, which +leaves the coefficients of its 1008 bytes of output and the AND of whether +each has 256 in `r14` (`P2`); and the epilogue, which returns it (`r4K`, as +`vg_mldsa_rej_ntt_poly`'s contract `rnK` for each seed). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Impl.MlDsa.X86_64.Sample.Rej4 (oJ half first second zeroJ rejNTT4Avx2) +open VG.Proof.MlKem.X86_64 (Keep ea_at add_ofNat_zero pR WP.keep ofNat64_pred sample4K) +open VG.Proof.MlKem.X86_64.S4 +open VG.Proof.MlDsa.Sample (rnFold Stored stored_nil stored_frame stored_polyIs toPoly G_length) +open VG.Spec.MlKem (poly4 seed4) +open VG.Spec.MlDsa (Zq G PolyIs) +open VG.Proof.MlKem (xofByte) + +/-- `vg_mldsa_rej_ntt_poly4(seeds = rdi, a = rsi, scratch = rdx) -> eax` +(either implementation), with 24 bytes of stack below `rsp`: for each seed, +what `vg_mldsa_rej_ntt_poly`'s contract `rnK` says. -/ +def r4K : Contract isa where + pre := sample4K.pre + post s s' := + (s'.gpr .rax).setWidth 32 = + (if (List.range 4).all (fun k => (rnFold [] (G (seed4 s.mem (s.gpr .rdi) k) 1008)).length == 256) + then 1 else 0) ∧ + ∀ k < 4, (rnFold [] (G (seed4 s.mem (s.gpr .rdi) k) 1008)).length = 256 → + PolyIs s'.mem (poly4 (s.gpr .rsi) k) (toPoly (rnFold [] (G (seed4 s.mem (s.gpr .rdi) k) 1008))) + pub := sample4K.pub + +/-- The coefficients of seed `k`'s whole output. -/ +theorem Lt_336 (σ : State) (k : Nat) : Lt σ k 336 = rnFold [] (G (B σ k) 1008) := by + simp only [Lt]; rw [List.take_of_length_le (by rw [Xb, G_length])] + +theorem Lt_zero (σ : State) (k : Nat) : Lt σ k 0 = [] := by + simp only [Lt, Nat.mul_zero, List.take_zero]; rfl + +/-- 1 if each of the first `K` seeds has 256 coefficients, 0 otherwise. -/ +def okN (σ : State) (K : Nat) : Nat := if (List.range K).all (fun k => (Lt σ k 336).length == 256) then 1 else 0 + +/-- Where the counts are kept. -/ +abbrev jR (σ : State) : Region := ⟨at' σ oJ, 32⟩ + +/-- The saved registers. -/ +abbrev svR (σ : State) : Region := ⟨at' σ oSave, 40⟩ + +section +variable {σ : State} (hp : Pre σ) +include hp + +/-! ## The regions -/ + +omit hp in +theorem low_jR : (lowR σ).Disjoint (jR σ) := + Offset.base_disjoint (scr σ) (k := oSave) (e := oJ) (n := 32) (by simp only [oSave, oJ]; omega) + (by simp only [oJ]; omega) + +omit hp in +theorem sv_jR : (svR σ).Disjoint (jR σ) := + Offset.disjoint (scr σ) (d := oSave) (n := 40) (e := oJ) (k := 32) (.inl (by simp only [oSave, oJ]; omega)) + (by simp only [oSave]; omega) (by simp only [oJ]; omega) + +theorem low_poly {K : Nat} (hK : K < 4) : (lowR σ).Disjoint (pR (poly4 (aP σ) K)) := + Region.Disjoint.sub_right (Region.Disjoint.sub_left hp.a_scr.symm (Region.sub_prefix (by simp only [oSave]; omega))) + (sub_poly hK) + +theorem sv_poly {K : Nat} (hK : K < 4) : (svR σ).Disjoint (pR (poly4 (aP σ) K)) := + scr_poly hp hK (a := oSave) (n := 40) (by simp only [oSave]; omega) _ (List.mem_singleton_self _) + +theorem jR_poly {K : Nat} (hK : K < 4) : (jR σ).Disjoint (pR (poly4 (aP σ) K)) := + scr_poly hp hK (a := oJ) (n := 32) (by simp only [oJ]; omega) _ (List.mem_singleton_self _) + +theorem poly_jR (k : Nat) (hk : k < 4) : (VG.Proof.MlDsa.Sample.polyR (poly4 (aP σ) k)).Disjoint (jR σ) := + (jR_poly hp hk).symm + +/-! ## Frames -/ + +omit hp in +/-- `Env` after writes apart from the saved registers, in the regions of the precondition. -/ +theorem env_frame {s s' : State} (he : EnvK σ s) {rs : List Region} (hf : Frame rs s.mem s'.mem) + (hsv : ∀ r ∈ rs, (svR σ).Disjoint r) (hsub : ∀ r ∈ rs, ∃ R ∈ [aR σ, scrR σ, stkR σ], Region.Sub r R) + (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15], s'.gpr r = s.gpr r) : + EnvK σ s' := by + refine ⟨hrd.trans he.rd, hwr.trans he.wr, by rw [hg .rbx (by simp), he.rbx], by rw [hg .r12 (by simp), he.r12], + by rw [hg .r13 (by simp), he.r13], by rw [hg .rsp (by simp), he.rsp], by rw [hg .r15 (by simp), he.r15], + fun i hi => ?_, he.frame.trans (hf.sub hsub)⟩ + rw [hf.readW (Region.contains_self _ _) (fun r hr => (hsv r hr).sub_left + (Offset.sub (scr σ) (d := oSave + 8 * i) (n := 8) (e := oSave) (k := 40) (by omega) (by omega))) (by decide)] + exact he.saved i hi + +omit hp in +/-- `SqT` after writes apart from the low scratch space and the saved registers. -/ +theorem sqT_frame {t n : Nat} (hn : n ≤ 3) {s s' : State} (h : SqT σ t n s) {rs : List Region} (hf : Frame rs s.mem s'.mem) + (hlow : ∀ r ∈ rs, (lowR σ).Disjoint r) (hsv : ∀ r ∈ rs, (svR σ).Disjoint r) + (hsub : ∀ r ∈ rs, ∃ R ∈ [aR σ, scrR σ, stkR σ], Region.Sub r R) (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) + (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15, .r14], s'.gpr r = s.gpr r) : SqT σ t n s' := by + refine ⟨env_frame h.env hf hsv hsub hrd hwr fun r hr => hg r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr ⊢; rcases hr with h | h | h | h | h <;> simp [h]), + by rw [hg .r14 (by simp), h.r14], fun r hr k hk => ?_, fun i hi k hk => ?_, fun k hk p hp' => ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (fun r' hr' => (hlow r' hr').sub_left + (Offset.sub_base (scr σ) (d := 32 * (50 + r) + 8 * k) (by simp only [oSave]; omega))) (by decide)] + exact h.rc r hr k hk + · rw [hf.readW (Region.contains_self _ _) (fun r' hr' => (hlow r' hr').sub_left + (Offset.sub_base (scr σ) (d := 32 * i + 8 * k) (by simp only [oSave]; omega))) (by decide)] + exact h.lanes i hi k hk + · rw [frame_byte hf (R := ⟨at' σ (oBuf + 504 * k + p), 1⟩) (fun r' hr' => (hlow r' hr').sub_left + (Offset.sub_base (scr σ) (d := oBuf + 504 * k + p) (by simp only [oBuf, oSave]; omega))) + (Region.contains_self _ _)] + exact h.buf k hk p (by omega) + +/-- A count, apart from writes to a polynomial. -/ +theorem j_poly {K : Nat} (hK : K < 4) {m m' : Mem} (hf : Frame [pR (poly4 (aP σ) K)] m m') {k : Nat} (hk : k < 4) : + m'.readW (at' σ (oJ + 8 * k)) 64 = m.readW (at' σ (oJ + 8 * k)) 64 := + hf.readW (Region.contains_self _ _) (scr_poly hp hK (a := oJ + 8 * k) (n := 8) (by simp only [oJ]; omega)) + (by decide) + +omit hp in +/-- A stored polynomial, apart from writes to another one. -/ +theorem st_poly {K k : Nat} (hK : K < 4) (hk : k < 4) (hne : k ≠ K) {m m' : Mem} + (hf : Frame [pR (poly4 (aP σ) K)] m m') {L : List Zq} (h : Stored m (poly4 (aP σ) k) L) (hL : L.length ≤ 256) : + Stored m' (poly4 (aP σ) k) L := + stored_frame hf (poly_poly hK hk hne) h hL + +omit hp in +theorem sub_polyR {K : Nat} (hK : K < 4) : ∀ r ∈ [pR (poly4 (aP σ) K)], ∃ R ∈ [aR σ, scrR σ, stkR σ], Region.Sub r R := + fun r hr => by rw [List.mem_singleton.mp hr]; exact ⟨aR σ, by simp, sub_poly hK⟩ + +omit hp in +theorem sub_jR : ∀ r ∈ [jR σ], ∃ R ∈ [aR σ, scrR σ, stkR σ], Region.Sub r R := + fun r hr => by rw [List.mem_singleton.mp hr]; exact ⟨scrR σ, by simp, sub_scr (by simp only [oJ]; omega)⟩ + +omit hp in +theorem jR_contains {k : Nat} (hk : k < 4) : (jR σ).Contains (at' σ (oJ + 8 * k)) 8 := + Offset.contains (scr σ) (d := oJ + 8 * k) (n := 8) (e := oJ) (k := 32) (by omega) (by omega) (by simp only [oJ]; omega) + +omit hp in +theorem jR_write {k : Nat} (hk : k < 4) {m m' : Mem} (hf : Frame [jR σ] m m') (v : BitVec 64) : + Frame [jR σ] m (m'.writeW (at' σ (oJ + 8 * k)) v) := + hf.writeW (List.mem_singleton_self _) v (jR_contains hk) + +end + +/-! ## The first halves -/ + +/-- After the first half of the seeds before `K`. -/ +structure P1 (σ : State) (K : Nat) (s : State) : Prop where + sq : SqT σ 0 3 s + j : ∀ k < 4, s.mem.readW (at' σ (oJ + 8 * k)) 64 = BitVec.ofNat 64 (Lt σ k (if k < K then 168 else 0)).length + st : ∀ k < 4, Stored s.mem (poly4 (aP σ) k) (Lt σ k (if k < K then 168 else 0)) + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +theorem zeroJ_eq : zeroJ = [.mov32 .rax (.imm 0), .store (at_ .rbx (oJ + 8 * 0)) .rax, .store (at_ .rbx (oJ + 8 * 1)) .rax, + .store (at_ .rbx (oJ + 8 * 2)) .rax, .store (at_ .rbx (oJ + 8 * 3)) .rax] := rfl + +/-- The counts zeroed. -/ +theorem zeroJ_ok {s : State} (h : SqT σ 0 3 s) : WP isa (.block zeroJ) s (P1 σ 0) := by + have hin : ∀ k < 4, InRegions s.wr (at' σ (oJ + 8 * k)) 8 := fun k hk => + in_scr hp h.env.wr (by simp only [oJ]; omega) + rw [zeroJ_eq] + refine WP.mono (WP.keep [.rax] (Q := fun s' => s'.mem = (((s.mem.writeW (at' σ (oJ + 8 * 0)) (0 : BitVec 64)).writeW + (at' σ (oJ + 8 * 1)) (0 : BitVec 64)).writeW (at' σ (oJ + 8 * 2)) (0 : BitVec 64)).writeW (at' σ (oJ + 8 * 3)) + (0 : BitVec 64)) + (by + have h0 := hin 0 (by decide); have h1 := hin 1 (by decide); have h2 := hin 2 (by decide) + have h3 := hin 3 (by decide) + xrun [h.env.rbx, h0, h1, h2, h3] + rfl) + (by decide)) fun s' ⟨hm, k⟩ => ?_ + have hf : Frame [jR σ] s.mem s'.mem := by + rw [hm] + exact jR_write (by decide) (jR_write (by decide) (jR_write (by decide) (jR_write (by decide) (Frame.refl _ _) _) _) _) _ + refine ⟨sqT_frame (by decide) h hf (fun r hr => by rw [List.mem_singleton.mp hr]; exact low_jR) + (fun r hr => by rw [List.mem_singleton.mp hr]; exact sv_jR) sub_jR k.2.1 k.2.2 + fun r hr => k.gpr (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide), fun k hk => ?_, fun k hk => ?_⟩ + · rw [hm, ite_eq_right (by omega), Lt_zero, List.length_nil] + rcases (by omega : k = 0 ∨ k = 1 ∨ k = 2 ∨ k = 3) with rfl | rfl | rfl | rfl <;> + simp (disch := omega) only [oJ, Mem.readW_writeW_self64, rd64_off, Nat.reduceMul, Nat.reduceAdd] <;> rfl + · rw [ite_eq_right (by omega), Lt_zero] + exact stored_nil _ _ + +omit hp in +theorem hpre1 {K : Nat} (hK : K < 4) {s : State} (h : P1 σ K s) : HPre σ K 0 s := + ⟨h.sq.env, fun p hp' => by rw [h.sq.buf K hK p (by omega)], + by rw [h.j K hK, ite_eq_right (by omega), Nat.mul_zero], + by have := h.st K hK; rw [ite_eq_right (by omega)] at this; rwa [Nat.mul_zero]⟩ + +/-- `j` kept, after the first half of seed `K`. -/ +theorem firstEnd_ok {K : Nat} (hK : K < 4) {s s₁ : State} (h : P1 σ K s) (hA : HAt σ s K 0 168 s₁) : + WP isa (.block [.store (at_ .rbx (oJ + 8 * K)) .rdi]) s₁ (P1 σ (K + 1)) := by + have hin : InRegions s₁.wr (scr σ + BitVec.ofNat 64 (oJ + 8 * K)) 8 := in_scr hp hA.env.wr (by simp only [oJ]; omega) + refine WP.mono (WP.keep [] (Q := fun s' => s'.mem = s₁.mem.writeW (at' σ (oJ + 8 * K)) (s₁.gpr .rdi)) + (by xrun [hA.env.rbx, hin]) rfl) fun s₂ ⟨hm, k⟩ => ?_ + have hf₂ : Frame [jR σ] s₁.mem s₂.mem := by + rw [hm]; exact jR_write hK (Frame.refl _ _) _ + have k₁ : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15, .r14], s₁.gpr r = s.gpr r := fun r hr => + hA.kp.gpr (by simp only [List.mem_cons, List.not_mem_nil, or_false] at hr; rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide) + refine ⟨sqT_frame (by decide) (sqT_frame (by decide) h.sq hA.fr (fun r hr => by rw [List.mem_singleton.mp hr]; exact low_poly hp hK) + (fun r hr => by rw [List.mem_singleton.mp hr]; exact sv_poly hp hK) (sub_polyR hK) hA.kp.2.1 hA.kp.2.2 k₁) + hf₂ (fun r hr => by rw [List.mem_singleton.mp hr]; exact low_jR) + (fun r hr => by rw [List.mem_singleton.mp hr]; exact sv_jR) sub_jR k.2.1 k.2.2 fun r hr => k.gpr (by simp), + fun k hk => ?_, fun k hk => ?_⟩ + · rw [hm] + by_cases e : k = K + · subst e + rw [Mem.readW_writeW_self64, hA.rdi, ite_eq_left (by omega)] + · rw [rd64_off (by simp only [oJ]; omega) (by simp only [oJ]; omega) (by omega), j_poly hp hK hA.fr hk, h.j k hk] + congr 3 + by_cases hk' : k < K + · rw [ite_eq_left hk', ite_eq_left (by omega)] + · rw [ite_eq_right hk', ite_eq_right (by omega)] + · have hL := Lt_length_le σ k (if k < K + 1 then 168 else 0) + refine stored_frame hf₂ (fun r hr => by rw [List.mem_singleton.mp hr]; exact poly_jR hp k hk) ?_ hL + by_cases e : k = K + · subst e + rw [ite_eq_left (by omega)] + exact hA.stored + · have := st_poly hK hk e hA.fr (h.st k hk) (Lt_length_le σ k _) + by_cases hk' : k < K + · rw [ite_eq_left hk'] at this; rwa [ite_eq_left (by omega)] + · rw [ite_eq_right hk'] at this; rwa [ite_eq_right (by omega)] + +/-- The first half of seed `K`. -/ +theorem first_ok {K : Nat} (hK : K < 4) {s : State} (h : P1 σ K s) : WP isa (first K) s (P1 σ (K + 1)) := + WP.seq (WP.mono (half_ok hp hK (by decide) (hpre1 hK h)) fun _ hA => firstEnd_ok hp hK h hA) + +end + + +/-! ## The second halves -/ + +/-- After the second half of the seeds before `K`. -/ +structure P2 (σ : State) (K : Nat) (s : State) : Prop where + env : EnvK σ s + r14 : s.gpr .r14 = BitVec.ofNat 64 (okN σ K) + buf : ∀ k < 4, ∀ p < 504, s.mem (at' σ (oBuf + 504 * k + p)) = xofByte (B σ k) (504 + p) + j : ∀ k < 4, K ≤ k → s.mem.readW (at' σ (oJ + 8 * k)) 64 = BitVec.ofNat 64 (Lt σ k 168).length + st : ∀ k < 4, Stored s.mem (poly4 (aP σ) k) (Lt σ k (if k < K then 336 else 168)) + +theorem tail_ok (s : State) : + WP isa (.block [.mov .rax (.reg .rdi), .shift .shr .rax 8, .alu32 .and .r14 (.reg .rax)]) s fun s' => + (s'.gpr .r14 = BitVec.setWidth 64 ((s.gpr .r14).setWidth 32 &&& ((s.gpr .rdi) >>> 8).setWidth 32) ∧ + s'.mem = s.mem) ∧ Keep [.rax, .r14] s s' := by + refine WP.keep _ ?_ (by decide) + xrun + +/-- Whether a count is 256, from its bit 8. -/ +theorem full_bit {n : Nat} (hn : n ≤ 256) : + (BitVec.ofNat 64 n >>> 8).setWidth 32 = if n == 256 then 1 else 0 := by + by_cases e : n = 256 + · subst e; decide + · rw [ite_eq_right (by simpa using e)] + apply BitVec.eq_of_toNat_eq + have h0 : (0 : BitVec 32).toNat = 0 := rfl + simp only [BitVec.toNat_setWidth, BitVec.toNat_ushiftRight, BitVec.toNat_ofNat, Nat.shiftRight_eq_div_pow] + rw [h0] + omega + +theorem okN_succ (σ : State) (K : Nat) : BitVec.setWidth 64 ((BitVec.ofNat 64 (okN σ K)).setWidth 32 &&& + (BitVec.ofNat 64 (Lt σ K 336).length >>> 8).setWidth 32) = BitVec.ofNat 64 (okN σ (K + 1)) := by + rw [full_bit (Lt_length_le σ K 336)] + simp only [okN, List.range_succ, List.all_append, List.all_cons, List.all_nil, Bool.and_true] + cases (List.range K).all fun k => (Lt σ k 336).length == 256 <;> + cases (Lt σ K 336).length == 256 <;> rfl + +section +variable {σ : State} (hp : Pre σ) +include hp + +/-- During the second squeezes: the counts and coefficients of the first halves. -/ +structure M2 (σ : State) (n : Nat) (s : State) : Prop where + sq : SqT σ 3 n s + j : ∀ k < 4, s.mem.readW (at' σ (oJ + 8 * k)) 64 = BitVec.ofNat 64 (Lt σ k 168).length + st : ∀ k < 4, Stored s.mem (poly4 (aP σ) k) (Lt σ k 168) + +omit hp in +theorem m2_of {s : State} (h : P1 σ 4 s) : M2 σ 0 s := + ⟨⟨h.sq.env, h.sq.r14, h.sq.rc, h.sq.lanes, fun _ _ p hp' => absurd hp' (by omega)⟩, + fun k hk => by rw [h.j k hk, ite_eq_left hk], fun k hk => by have := h.st k hk; rwa [ite_eq_left hk] at this⟩ + +/-- A second squeeze. -/ +theorem sqM_ok {n : Nat} (hn : n < 3) {s : State} (h : M2 σ n s) : WP isa (squeeze4 n) s (M2 σ (n + 1)) := by + refine WP.mono (sqT_ok hp hn h.sq) fun s' ⟨q, hf⟩ => ⟨q, fun k hk => ?_, fun k hk => ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (fun r hr => by + rw [List.mem_singleton.mp hr] + exact (Offset.base_disjoint (scr σ) (k := oSave) (e := oJ + 8 * k) (n := 8) (by simp only [oSave, oJ]; omega) + (by simp only [oJ]; omega)).symm) (by decide), h.j k hk] + · exact stored_frame hf (fun r hr => by rw [List.mem_singleton.mp hr]; exact (low_poly hp hk).symm) (h.st k hk) + (Lt_length_le σ k _) + +omit hp in +/-- `vzeroupper`, after the second squeezes. -/ +theorem vz_ok {s : State} (h : M2 σ 3 s) : WP isa (.block [.vop .vzeroupper]) s (P2 σ 0) := by + refine WP.mono (WP.keep [] (Q := fun s' => s'.mem = s.mem) (by xrun; rfl) rfl) fun s' ⟨hm, k⟩ => ?_ + refine ⟨h.sq.env.keep hm k (by simp), by rw [k.gpr (by simp), h.sq.r14]; rfl, fun k hk p hp' => ?_, + fun k hk _ => by rw [hm]; exact h.j k hk, fun k hk => ?_⟩ + · rw [hm, h.sq.buf k hk p (by omega)] + · rw [ite_eq_right (by omega), hm]; exact h.st k hk + +omit hp in +theorem hpre2 {K : Nat} (hK : K < 4) {s : State} (h : P2 σ K s) : HPre σ K 1 s := + ⟨h.env, fun p hp' => by rw [h.buf K hK p hp'], by rw [h.j K hK (Nat.le_refl _), Nat.mul_one], + by have := h.st K hK; rw [ite_eq_right (by omega)] at this; rwa [Nat.mul_one]⟩ + +/-- The AND of whether seed `K` has 256 coefficients, after its second half. -/ +theorem secondEnd_ok {K : Nat} (hK : K < 4) {s s₁ : State} (h : P2 σ K s) (hA : HAt σ s K 1 168 s₁) : + WP isa (.block [.mov .rax (.reg .rdi), .shift .shr .rax 8, .alu32 .and .r14 (.reg .rax)]) s₁ (P2 σ (K + 1)) := by + refine WP.mono (tail_ok s₁) fun s₂ ⟨⟨h14, hm⟩, k⟩ => ?_ + have hf : Frame [pR (poly4 (aP σ) K)] s.mem s₂.mem := by rw [hm]; exact hA.fr + have hrdi : s₁.gpr .rdi = BitVec.ofNat 64 (Lt σ K 336).length := hA.rdi + refine ⟨hA.env.keep hm k (by simp), ?_, fun k hk p hp' => ?_, fun k hk hKk => ?_, fun k hk => ?_⟩ + · rw [h14, hA.kp.gpr (by decide), h.r14, hrdi, okN_succ] + · rw [frame_byte hf (R := ⟨at' σ (oBuf + 504 * k + p), 1⟩) (scr_poly hp hK (by simp only [oBuf]; omega)) + (Region.contains_self _ _)] + exact h.buf k hk p hp' + · rw [hm, j_poly hp hK hA.fr hk, h.j k hk (by omega)] + · rw [hm] + by_cases e : k = K + · subst e + rw [ite_eq_left (by omega)] + exact hA.stored + · have := st_poly hK hk e hA.fr (h.st k hk) (Lt_length_le σ k _) + by_cases hk' : k < K + · rw [ite_eq_left hk'] at this; rwa [ite_eq_left (by omega)] + · rw [ite_eq_right hk'] at this; rwa [ite_eq_right (by omega)] + +/-- The second half of seed `K`. -/ +theorem second_ok {K : Nat} (hK : K < 4) {s : State} (h : P2 σ K s) : WP isa (second K) s (P2 σ (K + 1)) := + WP.seq (WP.mono (half_ok hp hK (by decide) (hpre2 hK h)) fun _ hA => secondEnd_ok hp hK h hA) + +/-- The return value, and the callee-saved registers restored. -/ +theorem end_ok {s : State} (h : P2 σ 4 s) : + WP isa (.block epi) s fun s' => r4K.post σ s' ∧ gprPreserved σ s' := by + have hin : ∀ i < 5, InRegions (s.rd ++ s.wr) (scr σ + BitVec.ofNat 64 (oSave + 8 * i)) 8 := fun i hi => + in_scr' hp h.env.rd h.env.wr (by simp only [oSave]; omega) + rw [epi_eq] + refine WP.mono (WP.keep [.rax, .r14, .r13, .r12, .rbp, .rbx] (Q := fun s' => s'.mem = s.mem ∧ + (s'.gpr .rax).setWidth 32 = (s.gpr .r14).setWidth 32 ∧ + s'.gpr .r14 = s.mem.readW (at' σ (oSave + 8 * 4)) 64 ∧ s'.gpr .r13 = s.mem.readW (at' σ (oSave + 8 * 3)) 64 ∧ + s'.gpr .r12 = s.mem.readW (at' σ (oSave + 8 * 2)) 64 ∧ s'.gpr .rbp = s.mem.readW (at' σ (oSave + 8 * 1)) 64 ∧ + s'.gpr .rbx = s.mem.readW (at' σ (oSave + 8 * 0)) 64) + (by + have h0 := hin 0 (by decide); have h1 := hin 1 (by decide); have h2 := hin 2 (by decide) + have h3 := hin 3 (by decide); have h4 := hin 4 (by decide) + simp only [oSave, Nat.reduceMul, Nat.reduceAdd] at h0 h1 h2 h3 h4 + xrun [h0, h1, h2, h3, h4, h.env.rbx] + exact ⟨rfl, rfl, rfl, rfl, rfl⟩) + (by decide)) fun s' ⟨⟨hm, hax, h14, h13, h12, hbp, hbx⟩, k⟩ => ?_ + refine ⟨⟨?_, fun k hk e => ?_⟩, fun r hr => ?_, ?_⟩ + · rw [hax, h.r14, okN] + simp only [Lt_336] + split <;> rfl + · rw [hm] + have := h.st k hk + rw [ite_eq_left hk, Lt_336] at this + exact stored_polyIs this e + · simp only [calleeSaved, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl | rfl + · rw [hbx]; exact h.env.saved 0 (by decide) + · rw [hbp]; exact h.env.saved 1 (by decide) + · rw [k.gpr (by decide)]; exact h.env.rsp + · rw [h12]; exact h.env.saved 2 (by decide) + · rw [h13]; exact h.env.saved 3 (by decide) + · rw [h14]; exact h.env.saved 4 (by decide) + · rw [k.gpr (by decide)]; exact h.env.r15 + · rw [hm] + exact h.env.frame.readW (Region.contains_self _ _) (by + simpa using ⟨hp.ret_a, hp.ret_scr, Offset.base_disjoint_below (σ.gpr .rsp) (n := 24) (k := 8) (by omega)⟩) + (by decide) + +/-- Everything after the prologue, the round constants and the padded seeds. -/ +theorem body_ok {s : State} (h : SqInv σ 0 s) : + WP isa (.seq (squeeze4 0) (.seq (squeeze4 1) (.seq (squeeze4 2) (.seq (.block zeroJ) + (.seq (first 0) (.seq (first 1) (.seq (first 2) (.seq (first 3) + (.seq (squeeze4 0) (.seq (squeeze4 1) (.seq (squeeze4 2) (.seq (.block [.vop .vzeroupper]) + (.seq (second 0) (.seq (second 1) (.seq (second 2) (.seq (second 3) (.block epi))))))))))))))))) s + fun s' => r4K.post σ s' ∧ gprPreserved σ s' := by + refine WP.seq (WP.mono (sqT_ok hp (by decide) (sqT_of h)) fun s₁ ⟨q₁, _⟩ => WP.seq (WP.mono + (sqT_ok hp (by decide) q₁) fun s₂ ⟨q₂, _⟩ => WP.seq (WP.mono (sqT_ok hp (by decide) q₂) fun s₃ ⟨q₃, _⟩ => + WP.seq (WP.mono (zeroJ_ok hp q₃) fun s₄ p₀ => ?_)))) + refine WP.seq (WP.mono (first_ok hp (by decide) p₀) fun _ p₁ => WP.seq (WP.mono (first_ok hp (by decide) p₁) + fun _ p₂ => WP.seq (WP.mono (first_ok hp (by decide) p₂) fun _ p₃ => + WP.seq (WP.mono (first_ok hp (by decide) p₃) fun _ p₄ => ?_)))) + refine WP.seq (WP.mono (sqM_ok hp (by decide) (m2_of p₄)) fun _ m₁ => WP.seq (WP.mono (sqM_ok hp (by decide) m₁) + fun _ m₂ => WP.seq (WP.mono (sqM_ok hp (by decide) m₂) fun _ m₃ => WP.seq (WP.mono (vz_ok m₃) fun _ r₀ => ?_)))) + exact WP.seq (WP.mono (second_ok hp (by decide) r₀) fun _ r₁ => WP.seq (WP.mono (second_ok hp (by decide) r₁) + fun _ r₂ => WP.seq (WP.mono (second_ok hp (by decide) r₂) fun _ r₃ => + WP.seq (WP.mono (second_ok hp (by decide) r₃) fun _ r₄ => end_ok hp r₄)))) + +end + +theorem correct (σ : State) (hs : r4K.pre σ) : + ∃ t s', Exec isa rejNTT4Avx2 σ t s' ∧ abiPreserved σ s' ∧ r4K.post σ s' := by + have hp := pre_of hs + obtain ⟨t, s', he, hF⟩ := WP.seq (WP.mono (start_ok hp) fun _ h => body_ok hp h) + exact ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he hF.2, hF.1⟩ + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Verified.lean new file mode 100644 index 000000000..ddb0be67b --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Verified.lean @@ -0,0 +1,79 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Scalar +import VerifiedGarbage.Proof.MlKem.X86_64.S4Verified + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4` and `vg_mldsa_rej_ntt_poly4_avx2`, verified + +Untrusted: everything here is checked by Lean. The contract of the proofs +(`r4K`) implies the shared one of `Spec/`: a seed with 256 coefficients in +the 1008 bytes both implementations sample from has them within those +bounds (`rejNTT_some`), and one without has none within the least bound +(`rejNTT_none`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 +open VG.Proof.MlDsa.Sample (rnFold rejNTT_some rejNTT_none) +open VG.Proof.MlDsa.X86_64.Sample (leakBytes_inj) +open VG.Spec.MlDsa (G) + +theorem seed4_eq : Spec.MlDsa.seed4 = Spec.MlKem.seed4 := rfl +theorem poly4_eq : Spec.MlDsa.poly4 = Spec.MlKem.poly4 := rfl + +theorem r4_post {s s' : State} (h : r4K.post s s') : + let r := (s'.gpr .rax).setWidth 32 + (r = 1 → ∀ k < 4, Spec.MlDsa.Reduced s'.mem (Spec.MlDsa.poly4 (s.gpr .rsi) k)) ∧ + ((r = 1 ∧ ∀ k < 4, ∃ b : Spec.MlDsa.Bounds, Spec.MlDsa.rejNTTPoly b.rejNTT + (Spec.MlDsa.seed4 s.mem (s.gpr .rdi) k) = some (Spec.MlDsa.polyAt s'.mem (Spec.MlDsa.poly4 (s.gpr .rsi) k))) ∨ + (r = 0 ∧ ∃ k < 4, Spec.MlDsa.rejNTTPoly Spec.MlDsa.minBounds.rejNTT + (Spec.MlDsa.seed4 s.mem (s.gpr .rdi) k) = none)) := by + obtain ⟨hr, hp⟩ := h + intro r + simp only [seed4_eq, poly4_eq] + by_cases hall : ((List.range 4).all fun k => + (rnFold [] (G (Spec.MlKem.seed4 s.mem (s.gpr .rdi) k) 1008)).length == 256) = true + · rw [ite_eq_left hall] at hr + have hs : ∀ k < 4, (rnFold [] (G (Spec.MlKem.seed4 s.mem (s.gpr .rdi) k) 1008)).length = 256 := fun k hk => by + simpa using List.all_eq_true.mp hall k (List.mem_range.mpr hk) + refine ⟨fun _ k hk => (hp k hk (hs k hk)).1, .inl ⟨hr, fun k hk => ⟨{ Spec.MlDsa.minBounds with rejNTT := 1008 }, ?_⟩⟩⟩ + show Spec.MlDsa.rejNTTPoly 1008 _ = _ + rw [rejNTT_some (hs k hk), (hp k hk (hs k hk)).2] + · rw [ite_eq_right hall] at hr + refine ⟨fun h1 => absurd (hr.symm.trans h1) (by decide), .inr ⟨hr, ?_⟩⟩ + simp only [List.all_eq_true, List.mem_range, not_forall, beq_iff_eq] at hall + obtain ⟨k, hk, hk'⟩ := hall + exact ⟨k, hk, rejNTT_none (B := 1008) (by decide) (by decide) hk'⟩ + +theorem rej4_verified (c : Prog isa) (hc : ∀ σ, r4K.pre σ → ∃ t s', Exec isa c σ t s' ∧ abiPreserved σ s' ∧ r4K.post σ s') + (ht : ConstantTime isa r4K.pre r4K.pub c) : Verified X86_64.target c (Spec.MlDsa.rejNTT4Contract X86_64.abi 24) := + Verified.of_correct hc ht + { pre := by sig_implies_pre [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, + X86_64.abi, X86_64.argRegs] + post := by + intro s s' _ h + sig_post [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, X86_64.abi, X86_64.argRegs] + exact r4_post h + pub := by + intro s₁ s₂ _ _ h + sig_pub [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, X86_64.abi, + X86_64.argRegs] at h + sig_split h + sig_reduce [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, X86_64.abi, + X86_64.argRegs] + sig_simp [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, X86_64.abi, + X86_64.argRegs] [Nat.forall_lt_succ_right, Nat.not_lt_zero, false_imp_iff, forall_const, true_and] + sig_and_intros + sig_close + all_goals first | with_reducible assumption | exact leakBytes_inj ‹_› + sat := by + sig_implies_sat [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, X86_64.abi, + X86_64.argRegs] [MlKem.X86_64.sample4Sat] using MlKem.X86_64.sample4Sat } + +theorem rejNTT4Avx2_verified : Verified X86_64.target Impl.MlDsa.X86_64.Sample.Rej4.rejNTT4Avx2 + (Spec.MlDsa.rejNTT4Contract X86_64.abi 24) := rej4_verified _ correct ct + +theorem rejNTT4_verified : Verified X86_64.target Impl.MlDsa.X86_64.Sample.Rej4.rejNTT4 + (Spec.MlDsa.rejNTT4Contract X86_64.abi 24) := rej4_verified _ correct_scalar ct_scalar + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBounded.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBounded.lean index 133c1036a..be850a2d1 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBounded.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBounded.lean @@ -117,7 +117,7 @@ theorem lat_step {t : Nat} {s : State} (ht : t < 544) (h : LAt σ t s) : WP isa (rbBody (etaOf σ)) s fun s' => LAt σ (t + 1) s' ∧ s'.zf = some (BitVec.ofNat 64 (544 - t) - 1 == 0) := by have hw : pR (σ.gpr .rdx) ∈ s.wr := by rw [h.env.wr, hp.2.1]; simp have hp' := spOk hp - refine WP.mono (rbBody_ok (eta hp) s (aP := σ.gpr .rdx) h.env.rbp h.rdi (Lt_length_le t) hw h.stored + refine WP.mono (rbBody_ok (eta hp) s (aP := σ.gpr .rdx) h.env.rbp h.rdi (Lt_length_le t) (.of_mem hw) h.stored (by rw [h.rsi, at_add]; exact inScrRd hp' h.env (by omega))) fun s' ⟨hdi, hst, hf, hsi, hcx, hz, hk⟩ => ?_ have ht1 : Lt σ (t + 1) = rbStep (etaOf σ) (Lt σ t) ((X σ).getD t 0) := by simp only [Lt] diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedCT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedCT.lean index a77f4b731..a3efd7896 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedCT.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedCT.lean @@ -117,7 +117,7 @@ structure BPre (s : State) (aP : Addr) (L : List Zq) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length ≤ 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L r0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 1 @@ -132,7 +132,7 @@ structure MPre (s : State) (aP : Addr) (L : List Zq) (z : Byte) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length ≤ 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L cf : s.cf = some (decide (L.length < 256)) rax : s.gpr .rax = BitVec.setWidth 64 z @@ -167,7 +167,7 @@ theorem load_ct (η : Nat) : RelCT isa (BRel η) (.block rbLoad) (R1 η) := by def R2 (η : Nat) (s₁ s₂ : State) : Prop := ∃ (aP : Addr) (L₁ L₂ : List Zq) (z₁ z₂ : Byte), L₁.length = L₂.length ∧ L₁.length ≤ 256 ∧ L₂.length ≤ 256 ∧ s₁.gpr .rbp = aP ∧ s₂.gpr .rbp = aP ∧ s₁.gpr .rdi = BitVec.ofNat 64 L₁.length ∧ - s₂.gpr .rdi = BitVec.ofNat 64 L₂.length ∧ pR aP ∈ s₁.wr ∧ pR aP ∈ s₂.wr ∧ Stored s₁.mem aP L₁ ∧ + s₂.gpr .rdi = BitVec.ofNat 64 L₂.length ∧ CoeffsWr s₁.wr aP ∧ CoeffsWr s₂.wr aP ∧ Stored s₁.mem aP L₁ ∧ Stored s₂.mem aP L₂ ∧ s₁.gpr .rax = BitVec.setWidth 64 z₁ ∧ s₂.gpr .rax = BitVec.setWidth 64 z₂ ∧ halfByteOk η (z₁.toNat / 16) = halfByteOk η (z₂.toNat / 16) ∧ s₁.gpr .rcx = s₂.gpr .rcx ∧ s₁.gpr .rsi = s₂.gpr .rsi @@ -176,7 +176,7 @@ def R2 (η : Nat) (s₁ s₂ : State) : Prop := def R3 (η : Nat) (s₁ s₂ : State) : Prop := ∃ aP L₁ L₂ b₁ b₂, L₁.length = L₂.length ∧ L₁.length ≤ 256 ∧ s₁.gpr .rbp = aP ∧ s₂.gpr .rbp = aP ∧ s₁.gpr .rdi = BitVec.ofNat 64 L₁.length ∧ - s₂.gpr .rdi = BitVec.ofNat 64 L₂.length ∧ pR aP ∈ s₁.wr ∧ pR aP ∈ s₂.wr ∧ Stored s₁.mem aP L₁ ∧ + s₂.gpr .rdi = BitVec.ofNat 64 L₂.length ∧ CoeffsWr s₁.wr aP ∧ CoeffsWr s₂.wr aP ∧ Stored s₁.mem aP L₁ ∧ Stored s₂.mem aP L₂ ∧ b₁ < 16 ∧ b₂ < 16 ∧ (s₁.gpr .rdx).setWidth 32 = BitVec.ofNat 32 b₁ ∧ (s₂.gpr .rdx).setWidth 32 = BitVec.ofNat 32 b₂ ∧ halfByteOk η b₁ = halfByteOk η b₂ ∧ s₁.cf = some (decide (L₁.length < 256)) ∧ s₂.cf = some (decide (L₂.length < 256)) ∧ @@ -333,7 +333,7 @@ def LI (η n : Nat) (s₁ s₂ : State) : Prop := theorem bpre {σ : State} (hp : rbK.pre σ) {t : Nat} (ht : t < 544) {s : State} (h : LAt σ t s) : BPre s (σ.gpr .rdx) (Lt σ t) := - ⟨h.env.rbp, h.rdi, Lt_length_le t, by rw [h.env.wr, hp.2.1]; simp, h.stored, + ⟨h.env.rbp, h.rdi, Lt_length_le t, .of_mem (by rw [h.env.wr, hp.2.1]; simp), h.stored, by rw [h.rsi, at_add]; exact inScrRd (spOk hp) h.env (by omega)⟩ theorem li_brel {η n : Nat} {s₁ s₂ : State} (h : LI η n s₁ s₂) : BRel η s₁ s₂ := by diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedLoop.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedLoop.lean index cdf007305..1342c5570 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedLoop.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedLoop.lean @@ -152,7 +152,7 @@ theorem hbTry_length_le {η : Nat} {L : List Zq} (hL : L.length < 256) (b : Nat) /-- The second try, if `j < 256`. -/ theorem rbMid2_ok {η : Nat} (hη : η = 2 ∨ η = 4) (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (hcf : s.cf = some (decide ((s.gpr .rdi).toNat < 256))) {b : Nat} (hb : b < 16) (hdx : (s.gpr .rdx).setWidth 32 = BitVec.ofNat 32 b) : WP isa (.ite .b (rbTry η) (.block [])) s fun s' => @@ -169,7 +169,7 @@ theorem rbMid2_ok {η : Nat} (hη : η = 2 ∨ η = 4) (s : State) {aP : Addr} { /-- The two tries, if `j < 256`. -/ theorem rbMid_ok {η : Nat} (hη : η = 2 ∨ η = 4) (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (hcf : s.cf = some (decide ((s.gpr .rdi).toNat < 256))) {z : Byte} (hax : s.gpr .rax = BitVec.setWidth 64 z) (hdx : (s.gpr .rdx).setWidth 32 = BitVec.ofNat 32 (z.toNat % 16)) : WP isa (.ite .b (.seq (rbTry η) (.seq (.block rbHi) (.ite .b (rbTry η) (.block [])))) (.block [])) s @@ -196,7 +196,7 @@ theorem sx1' : BitVec.signExtend 64 (1 : BitVec 32) = BitVec.ofNat 64 1 := by de /-- An iteration: what `rbStep` does to the coefficients `L`. -/ theorem rbBody_ok {η : Nat} (hη : η = 2 ∨ η = 4) (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 1) : WP isa (rbBody η) s fun s' => s'.gpr .rdi = BitVec.ofNat 64 (rbStep η L (s.mem (s.gpr .rsi))).length ∧ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNtt.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNtt.lean index fe55b6adf..341329234 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNtt.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNtt.lean @@ -112,7 +112,7 @@ theorem Lt_length_le (t : Nat) : (Lt σ t).length ≤ 256 := rnFold_length_le (b theorem lat_step {t : Nat} {s : State} (ht : t < 336) (h : LAt σ t s) : WP isa rnBody s fun s' => LAt σ (t + 1) s' ∧ s'.zf = some (BitVec.ofNat 64 (336 - t) - 1 == 0) := by have hw : pR (σ.gpr .rsi) ∈ s.wr := by rw [h.env.wr, hp.2.1]; simp - refine WP.mono (rnBody_ok s (aP := σ.gpr .rsi) h.env.rbp h.rdi (Lt_length_le t) hw h.stored + refine WP.mono (rnBody_ok s (aP := σ.gpr .rsi) h.env.rbp h.rdi (Lt_length_le t) (.of_mem hw) h.stored (by simpa using lat_regions hp h (k := 0) (by omega)) (lat_regions hp h (by omega)) (lat_regions hp h (by omega))) fun s' ⟨hdi, hst, hf, hsi, hcx, hz, hk⟩ => ?_ have e0 := out_byte h (k := 0) (by omega) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttCT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttCT.lean index a8f88337d..ff749eb6f 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttCT.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttCT.lean @@ -57,7 +57,7 @@ structure MPre (s : State) (aP : Addr) (L : List Zq) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length ≤ 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L cf : s.cf = some (decide ((s.gpr .rdi).toNat < 256)) @@ -127,7 +127,7 @@ structure BPre (s : State) (aP : Addr) (L : List Zq) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length ≤ 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L r0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 1 r1 : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 1) 1 @@ -196,7 +196,7 @@ def LI (n : Nat) (s₁ s₂ : State) : Prop := theorem bpre {σ : State} (hp : rnK.pre σ) {t : Nat} (ht : t < 336) {s : State} (h : LAt σ t s) : BPre s (σ.gpr .rsi) (Lt σ t) := - ⟨h.env.rbp, h.rdi, Lt_length_le t, by rw [h.env.wr, hp.2.1]; simp, h.stored, + ⟨h.env.rbp, h.rdi, Lt_length_le t, .of_mem (by rw [h.env.wr, hp.2.1]; simp), h.stored, by simpa using lat_regions hp h (k := 0) (by omega), lat_regions hp h (by omega), lat_regions hp h (by omega)⟩ theorem li_brel {n : Nat} {s₁ s₂ : State} (h : LI n s₁ s₂) : BRel s₁ s₂ := by diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttLoop.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttLoop.lean index f0b9a5d52..c782b78b3 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttLoop.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttLoop.lean @@ -68,14 +68,21 @@ theorem storeJ_ok (r : Reg) (s : State) {a : Addr} (ha : s.ea aJ = a) (hw : InRe refine WP.keep _ ?_ (by rfl) xrun [ha, hw] +/-- Each coefficient of the polynomial at `aP` lies in one of the writable +regions `wr`. -/ +def CoeffsWr (wr : List Region) (aP : Addr) : Prop := ∀ i < 256, InRegions wr (coeffAddr aP i) 4 + +theorem CoeffsWr.of_mem {wr : List Region} {aP : Addr} (h : pR aP ∈ wr) : CoeffsWr wr aP := + fun _ hi => ⟨_, h, coeff_contains _ hi⟩ + /-- Storing the word `v` (less than `q`) as the next coefficient. -/ theorem store_next {s : State} {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length < 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length < 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (r : Reg) {v : BitVec 32} (hv : (s.gpr r).setWidth 32 = v) (hq : v.toNat < q) : WP isa (.block [.store32 aJ r, .alu .add .rdi (.imm 1)]) s fun s' => s'.gpr .rdi = BitVec.ofNat 64 (L ++ [Fin.ofNat q v.toNat]).length ∧ Stored s'.mem aP (L ++ [Fin.ofNat q v.toNat]) ∧ Frame [pR aP] s.mem s'.mem ∧ Keep [.rdi] s s' := by - refine WP.mono (storeJ_ok r s (ea_aJ s hbp hdi) ⟨_, hw, coeff_contains _ hL⟩) fun s' ⟨⟨hm, hdi'⟩, k⟩ => ?_ + refine WP.mono (storeJ_ok r s (ea_aJ s hbp hdi) (hw _ hL)) fun s' ⟨⟨hm, hdi'⟩, k⟩ => ?_ have hz : zw (Fin.ofNat q v.toNat) = v := by apply BitVec.eq_of_toNat_eq rw [zw_toNat, Fin.val_ofNat, Nat.mod_eq_of_lt hq] @@ -90,7 +97,7 @@ structure TryPre (s : State) (aP : Addr) (L : List Zq) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length < 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L /-- The coefficients after a try of the value `v`. -/ @@ -124,7 +131,7 @@ theorem rnStep_eq (L : List Zq) (b₀ b₁ b₂ : Byte) : rnStep L b₀ b₁ b /-- The try, if `j < 256`. -/ theorem rnMid_ok (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (hcf : s.cf = some (decide ((s.gpr .rdi).toNat < 256))) : WP isa (.ite .b rnTry (.block [])) s fun s' => s'.gpr .rdi = BitVec.ofNat 64 (rnMid L ((s.gpr .r8).setWidth 32).toNat).length ∧ @@ -147,7 +154,7 @@ theorem sx3 : BitVec.signExtend 64 (3 : BitVec 32) = BitVec.ofNat 64 3 := by dec /-- An iteration: what `rnStep` does to the coefficients `L`. -/ theorem rnBody_ok (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 1) (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 1) 1) (h2 : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 2) 1) : diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Base.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Base.lean index a1666ba22..46f211eed 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Base.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Base.lean @@ -20,7 +20,7 @@ Untrusted: everything here is checked by Lean. checked by evaluation (`inB`, `sepB`). * What a piece of code leaves (`PostB`): the permissions, the registers of the layout and the stack pointer, and memory but within the regions it - writes and the 24 bytes of stack below `rsp` (its calls' return + writes and the 32 bytes of stack below `rsp` (its calls' return addresses). * A call of verified code, with the moves of its arguments before it (`callAt_ok`, `callAt_tr`). @@ -121,7 +121,7 @@ structure Post (s s' : State) (W : List Region) : Prop where rd : s'.rd = s.rd wr : s'.wr = s.wr cs : ∀ r ∈ calleeSaved, s'.gpr r = s.gpr r - frame : Frame (W ++ [below (s.gpr .rsp) 24]) s.mem s'.mem + frame : Frame (W ++ [below (s.gpr .rsp) 32]) s.mem s'.mem /-- What a piece of code leaves: the permissions, the registers `bases` and the stack pointer, and memory but within `W` and the stack. -/ @@ -130,7 +130,7 @@ structure PostB (s s' : State) (W : List Region) : Prop where wr : s'.wr = s.wr bs : ∀ r ∈ bases, s'.gpr r = s.gpr r rsp : s'.gpr .rsp = s.gpr .rsp - frame : Frame (W ++ [below (s.gpr .rsp) 24]) s.mem s'.mem + frame : Frame (W ++ [below (s.gpr .rsp) 32]) s.mem s'.mem theorem Post.rsp {s s' : State} {W : List Region} (h : Post s s' W) : s'.gpr .rsp = s.gpr .rsp := h.cs .rsp (by decide) @@ -259,13 +259,13 @@ structure Lay (rbs wbs : List (Reg × Nat)) (s : State) : Prop where small : ∀ b ∈ rbs ++ wbs, b.2 < 2 ^ 31 dj : ∀ b ∈ rbs ++ wbs, ∀ b' ∈ rbs ++ wbs, b.1 ≠ b'.1 → (b.1 ∈ wRegs ∨ b'.1 ∈ wRegs) → Region.Disjoint ⟨s.gpr b.1, b.2⟩ ⟨s.gpr b'.1, b'.2⟩ - stk : ∀ b ∈ rbs ++ wbs, (below (s.gpr .rsp) 24).Disjoint ⟨s.gpr b.1, b.2⟩ + stk : ∀ b ∈ rbs ++ wbs, (below (s.gpr .rsp) 32).Disjoint ⟨s.gpr b.1, b.2⟩ nw : ∀ b ∈ rbs ++ wbs, (s.gpr b.1).toNat + b.2 ≤ 2 ^ 64 rd : ∀ b ∈ rbs ++ wbs, InRegions (s.rd ++ s.wr) (s.gpr b.1) b.2 wr : ∀ b ∈ wbs, InRegions s.wr (s.gpr b.1) b.2 ret : ∀ b ∈ rbs ++ wbs, (Region.mk (s.gpr .rsp) 8).Disjoint ⟨s.gpr b.1, b.2⟩ bs : ∀ b ∈ rbs ++ wbs, b.1 ∈ bases - sp24 : 24 ≤ (s.gpr .rsp).toNat + sp32 : 32 ≤ (s.gpr .rsp).toNat section variable {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) @@ -292,7 +292,7 @@ theorem Lay.disj {p q : Ptr} {l k : Nat} (h : sepB (rbs ++ wbs) p l q k = true) · exact (off_disj h2 (by omega)).symm theorem Lay.stkD {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : - (below (s.gpr .rsp) 24).Disjoint ⟨pa s p, l⟩ := by + (below (s.gpr .rsp) 32).Disjoint ⟨pa s p, l⟩ := by obtain ⟨n, hn, hsub⟩ := L.sub h exact (L.stk _ hn).sub_right hsub @@ -327,7 +327,7 @@ theorem Lay.cW {p : Ptr} {l : Nat} (h : inB wbs p l = true) : Covers [⟨pa s p, theorem Lay.post {s' : State} {W : List Region} (hP : PostB s s' W) : Lay rbs wbs s' := by have e : ∀ b ∈ rbs ++ wbs, s'.gpr b.1 = s.gpr b.1 := fun b hb => hP.bs _ (L.bs b hb) refine ⟨L.small, fun b hb b' hb' hne hw => ?_, fun b hb => ?_, fun b hb => ?_, fun b hb => ?_, fun b hb => ?_, - fun b hb => ?_, L.bs, by rw [hP.rsp]; exact L.sp24⟩ + fun b hb => ?_, L.bs, by rw [hP.rsp]; exact L.sp32⟩ · rw [e b hb, e b' hb']; exact L.dj b hb b' hb' hne hw · rw [e b hb, hP.rsp]; exact L.stk b hb · rw [e b hb]; exact L.nw b hb @@ -388,7 +388,7 @@ variable {rbs wbs : List (Reg × Nat)} {s s' : State} (L : Lay rbs wbs s) {ws : include L theorem Lay.fdisj (hc : keepB (rbs ++ wbs) ws p l = true) : - ∀ r ∈ ws.map (toR s) ++ [below (s.gpr .rsp) 24], Region.Disjoint ⟨pa s p, l⟩ r := by + ∀ r ∈ ws.map (toR s) ++ [below (s.gpr .rsp) 32], Region.Disjoint ⟨pa s p, l⟩ r := by simp only [keepB, Bool.and_eq_true, List.all_eq_true] at hc obtain ⟨⟨_, hin⟩, hall⟩ := hc intro r hr diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Blocks.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Blocks.lean index ceeabaf25..77da47651 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Blocks.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Blocks.lean @@ -119,18 +119,24 @@ theorem maskBody_ok (s : State) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rdi) 4) ( refine WP.keep _ ?_ (by decide) xrun [h0, h1] -/-- The mask of the polynomial at `a` by `eax`: each coefficient `∧ -eax`. -/ -theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a : Ptr} - (hi : inB (rbs ++ wbs) a 1024 = true) (hw : inB wbs a 1024 = true) : - WP isa (mask a) s fun s' => PPostB s s' [(a, 1024)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ - ∀ i < n, coeffAt s'.mem (pa s a) i = coeffAt s.mem (pa s a) i &&& (0 - (s.gpr .rax).setWidth 32) := by +theorem coeffAt_writeW' (m : Mem) (p : Addr) {N i j : Nat} (hN : 4 * N ≤ 2 ^ 64) (hi : i < N) (hj : j < N) + (v : BitVec 32) : + coeffAt (m.writeW (p + BitVec.ofNat 64 (4 * j)) v) p i = if j = i then v else coeffAt m p i := by + split + · subst j; exact Mem.readW_writeW_self32 m _ v + · exact Mem.readW_writeW_sep (Offset.sep p (by omega) (by omega) (by omega)) (by decide) + +/-- The mask of the `N` coefficients from `a` by `eax`: each `∧ -eax`. -/ +theorem maskN_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a : Ptr} {N : Nat} (hN0 : 0 < N) + (hN : N < 2 ^ 29) (hi : inB (rbs ++ wbs) a (4 * N) = true) (hw : inB wbs a (4 * N) = true) : + WP isa (mask a N) s fun s' => PPostB s s' [(a, 4 * N)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ + ∀ i < N, coeffAt s'.mem (pa s a) i = coeffAt s.mem (pa s a) i &&& (0 - (s.gpr .rax).setWidth 32) := by have hS := L.ok - have hok : ∀ x ∈ ([(.rdi, .ptr a), (.rcx, .imm 256)] : List (Reg × Arg)), x.2.Ok ∧ x.1 ∈ argRegs := by + have hok : ∀ x ∈ ([(.rdi, .ptr a), (.rcx, .imm N)] : List (Reg × Arg)), x.2.Ok ∧ x.1 ∈ argRegs := by simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] - exact ⟨⟨ptr_ok hS hi, by decide⟩, ⟨show 256 < 2 ^ 31 by decide, by decide⟩⟩ + exact ⟨⟨ptr_ok hS hi, by decide⟩, ⟨show N < 2 ^ 31 by omega, by decide⟩⟩ have hrd := L.inR hi have hwr := L.inW hw - have nn : n = 256 := rfl unfold mask refine WP.seq ?_ rw [WP.block_append_iff] @@ -142,14 +148,14 @@ theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a simp only [List.mem_singleton]; intro h; rw [h] at hb; exact absurd hb (by decide) have e1 : s1.gpr .rdi = pa s a := by rw [hv1 _ (List.mem_cons_self ..)]; simp only [Arg.val, pa]; rw [k₀.gpr nb] - have e2 : s1.gpr .rcx = BitVec.ofNat 64 256 := hv1 _ (List.mem_cons_of_mem _ (List.mem_cons_self ..)) + have e2 : s1.gpr .rcx = BitVec.ofNat 64 N := hv1 _ (List.mem_cons_of_mem _ (List.mem_cons_self ..)) have hd1 : (s1.gpr .rdx).setWidth 32 = 0 - (s.gpr .rax).setWidth 32 := by rw [k1.gpr (by simp), hd₀] have k01 : Keep [.rdx, .rdi, .rcx] s s1 := (k₀.trans k1).mono (by simp) have hm01 : s1.mem = s.mem := hm1.trans hm₀ - refine WP.mono (wp_countdown (cnt := .rcx) (N := 256) (by decide) (by decide) (fun k s' => + refine WP.mono (wp_countdown (cnt := .rcx) (N := N) (by omega) hN0 (fun k s' => s'.gpr .rdi = pa s a + BitVec.ofNat 64 (4 * k) ∧ (s'.gpr .rdx).setWidth 32 = 0 - (s.gpr .rax).setWidth 32 ∧ - Frame [⟨pa s a, 1024⟩] s.mem s'.mem ∧ - (∀ i < n, coeffAt s'.mem (pa s a) i = + Frame [⟨pa s a, 4 * N⟩] s.mem s'.mem ∧ + (∀ i < N, coeffAt s'.mem (pa s a) i = if i < k then coeffAt s.mem (pa s a) i &&& (0 - (s.gpr .rax).setWidth 32) else coeffAt s.mem (pa s a) i) ∧ Keep [.rdx, .rdi, .rcx, .rax] s s') (fun k hk s' ⟨hdi, hdx, hf, hc, kk⟩ _ => ?_) (fun _ h => h) @@ -169,7 +175,7 @@ theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a have := hc k (by omega) rw [ifn (Nat.lt_irrefl _)] at this exact this - rw [hm, hdi, hck, hdx, Proof.MlDsa.Verify.coeffAt_writeW _ _ hi (by omega)] + rw [hm, hdi, hck, hdx, coeffAt_writeW' _ _ (N := N) (by omega) hi (by omega)] by_cases e : k = i · subst e; rw [ifp rfl, ifp (Nat.lt_succ_self _)] · rw [ifn e, hc i hi] @@ -177,4 +183,11 @@ theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a · rw [ifp h', ifp (by omega)] · rw [ifn h', ifn (by omega)] +/-- The mask of the polynomial at `a` by `eax`: each coefficient `∧ -eax`. -/ +theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a : Ptr} + (hi : inB (rbs ++ wbs) a 1024 = true) (hw : inB wbs a 1024 = true) : + WP isa (mask a) s fun s' => PPostB s s' [(a, 1024)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ + ∀ i < n, coeffAt s'.mem (pa s a) i = coeffAt s.mem (pa s a) i &&& (0 - (s.gpr .rax).setWidth 32) := + maskN_ok L (N := 256) (by decide) (by decide) hi hw + end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTBase.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTBase.lean index 1168a716f..8d1c9c257 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTBase.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTBase.lean @@ -67,13 +67,13 @@ theorem copy_tr {dst src : Ptr} {n : Nat} (hok : ∀ a ∈ copyArgs dst src n, a · rw [hx.1.1 _ (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))), hy.1.1 _ (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..)))]; rfl -theorem maskPre_wp {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) (s : State) : +theorem maskPre_wp {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) {N : Nat} (hN : N < 2 ^ 31) (s : State) : WP isa (.block (([.mov32 .rdx (.imm 0), .alu32 .sub .rdx (.reg .rax)] : List Instr) ++ - glue [(.rdi, .ptr a), (.rcx, .imm 256)])) s - fun s' => s'.gpr .rdi = (Arg.ptr a).val s ∧ s'.gpr .rcx = BitVec.ofNat 64 256 := by - have hok' : ∀ x ∈ ([(.rdi, .ptr a), (.rcx, .imm 256)] : List (Reg × Arg)), x.2.Ok ∧ x.1 ∈ argRegs := by + glue [(.rdi, .ptr a), (.rcx, .imm N)])) s + fun s' => s'.gpr .rdi = (Arg.ptr a).val s ∧ s'.gpr .rcx = BitVec.ofNat 64 N := by + have hok' : ∀ x ∈ ([(.rdi, .ptr a), (.rcx, .imm N)] : List (Reg × Arg)), x.2.Ok ∧ x.1 ∈ argRegs := by simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] - exact ⟨⟨hok, by decide⟩, ⟨show 256 < 2 ^ 31 by decide, by decide⟩⟩ + exact ⟨⟨hok, by decide⟩, ⟨hN, by decide⟩⟩ rw [WP.block_append_iff] refine WP.mono (maskPre_ok s) fun s₀ ⟨_, k₀⟩ => ?_ refine WP.mono (glue_ok _ hok' (by simp only [List.map_cons, List.map_nil]; decide) s₀) fun s1 ⟨⟨hv1, _⟩, _⟩ => @@ -84,10 +84,11 @@ theorem maskPre_wp {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) (s : St simp only [Arg.val, pa] rw [k₀.gpr nb] -theorem mask_tr {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) {P : State → State → Prop} - (hP : ∀ x y, P x y → SameB x y) : RelCT isa P (mask a) fun _ _ => True := by +theorem mask_tr {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) {N : Nat} (hN : N < 2 ^ 31) + {P : State → State → Prop} (hP : ∀ x y, P x y → SameB x y) : RelCT isa P (mask a N) fun _ _ => True := by unfold mask - refine blockLoop_tr (fun i hi s => ?_) (fun x y _ => ⟨maskPre_wp hok hb x, maskPre_wp hok hb y⟩) [.rdi, .rcx] + refine blockLoop_tr (fun i hi s => ?_) (fun x y _ => ⟨maskPre_wp hok hb hN x, maskPre_wp hok hb hN y⟩) + [.rdi, .rcx] (fun x y x' y' hp hx hy r hr => ?_) (by taint_decide) · rcases List.mem_append.mp hi with h | h · simp only [List.mem_cons, List.not_mem_nil, or_false] at h; rcases h with rfl | rfl <;> rfl diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean index 3c5c9a2c1..de99c2d5f 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean @@ -40,7 +40,7 @@ theorem aOne_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r have hkl := kl_le p hp simp only [aChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, Bool.or_eq_true, Bool.not_eq_true', decide_eq_false_iff_not, Nat.not_lt] at hck - obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hck + obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hck have hT : ∀ σ s, IA p r c σ s → T p σ s := fun _ _ h => h.t refine relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (aOne_ok C hp hv hr hc hs) fun _ h' => ⟨h, hh, h'⟩) ?_ have em : (8 * r + c) % 8 = c := by omega @@ -64,17 +64,94 @@ theorem aOne_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r exact ⟨WP.mono (rejNttAt_ok C.rejNtt L.1 hrej) fun _ h' => ⟨_, h'.1⟩, WP.mono (rejNttAt_ok C.rejNtt L.2.1 hrej) fun _ h' => ⟨_, h'.1⟩⟩ +theorem bytes136 (m : Mem) (P : Addr) : bytesAt m P 136 = bytesAt m P 34 ++ bytesAt m (P + BitVec.ofNat 64 34) 34 ++ + bytesAt m (P + BitVec.ofNat 64 68) 34 ++ bytesAt m (P + BitVec.ofNat 64 102) 34 := by + rw [show 136 = 34 + 102 from rfl, Proof.MlKem.bytesAt_add, show 102 = 34 + 68 from rfl, Proof.MlKem.bytesAt_add, + show 68 = 34 + 34 from rfl, Proof.MlKem.bytesAt_add] + simp only [BitVec.add_assoc, ← BitVec.ofNat_add, List.append_assoc, Nat.reduceAdd] + +/-- The four seeds of `SB4`, for the entries `(r, c₀), …, (r, c₀ + 3)`. -/ +theorem GS.seeds {p : Params} {h : List (Vector Bool n)} {r c c₀ : Nat} {σ s : State} (g : GS p h r c c₀ 4 σ s) : + bytesAt s.mem (pa s (sc oSB4)) 136 = aSeed (vPk p σ) r (c₀ + 0) ++ aSeed (vPk p σ) r (c₀ + 1) ++ + aSeed (vPk p σ) r (c₀ + 2) ++ aSeed (vPk p σ) r (c₀ + 3) := by + have e : ∀ k, pa s (sc (oSB4 + 34 * k)) = pa s (sc oSB4) + BitVec.ofNat 64 (34 * k) := fun k => by + simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add] + have b0 : bytesAt s.mem (pa s (sc oSB4)) 34 = aSeed (vPk p σ) r (c₀ + 0) := g.done 0 (by decide) + have b1 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 34) 34 = aSeed (vPk p σ) r (c₀ + 1) := by + have := g.done 1 (by decide); rw [e] at this; exact this + have b2 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 68) 34 = aSeed (vPk p σ) r (c₀ + 2) := by + have := g.done 2 (by decide); rw [e] at this; exact this + have b3 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 102) 34 = aSeed (vPk p σ) r (c₀ + 3) := by + have := g.done 3 (by decide); rw [e] at this; exact this + rw [bytes136, b0, b1, b2, b3] + +theorem slotBlock_tr {r c₀ j : Nat} {P : State → State → Prop} (hP : ∀ s₁ s₂, P s₁ s₂ → s₁.gpr .rbx = s₂.gpr .rbx) : + RelCT isa P (.block (setSR r c₀ j)) fun _ _ => True := + taintRel [.rbx] (fun s₁ s₂ h r hr => by simp only [List.mem_singleton] at hr; subst hr; exact hP s₁ s₂ h) + (hc := .block []) (by with_unfolding_all rfl) + +/-- The bytes of seed `j` of `SB4`, from `GS j` to `GS (j + 1)`. -/ +theorem slot_tr {p : Params} (hp : p ∈ params) {r c c₀ j : Nat} (hr : r < 256) (hj : j < 4) (hx : c₀ + j < 256) + (hck : slotChk p r c j = true) : + RelCT isa (RV p fun σ s => ∃ h, HN p σ h ∧ GS p h r c c₀ j σ s) (.block (setSR r c₀ j)) + (RV p fun σ s => ∃ h, HN p σ h ∧ GS p h r c c₀ (j + 1) σ s) := + relInv (fun σ s hv ⟨h, hh, g⟩ => WP.mono (slot_ok hp hv hr hj hx hck g) fun _ g' => ⟨h, hh, g'⟩) + (slotBlock_tr fun x y h => (RV.lrel hp (fun _ _ h => let ⟨_, _, g⟩ := h; g.s3.t) h).2.2.1 .rbx (by decide)) + +theorem aGrp_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r c₀ c : Nat} (hr : r < p.k) + (hck : gChk p r c₀ c = true) : RelCT isa (RV p (IA p r c)) (aGrp P p r c₀) (RV p (IA p r (c₀ + 4))) := by + have hkl := kl_le p hp + have hck' := hck + simp only [gChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, decide_eq_true_eq] at hck' + obtain ⟨⟨⟨⟨⟨⟨⟨hsl, hrej⟩, hin⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hck' + have hT : ∀ σ s, IA p r c σ s → T p σ s := fun _ _ h => h.t + have hTG : ∀ σ s, (∃ h, HN p σ h ∧ GS p h r c c₀ 4 σ s) → T p σ s := fun _ _ ⟨_, _, g⟩ => g.s3.t + refine relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (aGrp_ok C hp hv hr hck hs) fun _ h' => ⟨h, hh, h'⟩) ?_ + unfold aGrp + refine RelCT.seq (RelCT.mono (slot_tr hp (by omega) (by decide) (by omega) (hsl 0 (by decide))) + (fun x y ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁⟩, ⟨h₂, hh₂, s₂⟩⟩ => ⟨σ₁, σ₂, v₁, v₂, pub, + ⟨h₁, hh₁, s₁, fun _ h => absurd h (by omega)⟩, ⟨h₂, hh₂, s₂, fun _ h => absurd h (by omega)⟩⟩) + fun _ _ h => h) ?_ + refine RelCT.seq (slot_tr hp (by omega) (by decide) (by omega) (hsl 1 (by decide))) ?_ + refine RelCT.seq (slot_tr hp (by omega) (by decide) (by omega) (hsl 2 (by decide))) ?_ + refine RelCT.seq (slot_tr hp (by omega) (by decide) (by omega) (hsl 3 (by decide))) ?_ + unfold sampled4 + refine RelCT.seq (RelCT.sameB (rej4At_tr C.rej4 (layOk p hp) hrej fun x y h => ?_) (fun x y h => ?_) + (fun x y h => (RV.lrel hp hTG h).2.2)) + (sampledTail4_tr (ptr_ok (layOk p hp) hin) (show Reg.rbx ∈ bases by decide)) + · have L := RV.lrel hp hTG h + obtain ⟨σ₁, σ₂, _, _, pub, ⟨_, _, gx⟩, ⟨_, _, gy⟩⟩ := h + exact ⟨L.1, L.2.1, L.2.2, by rw [gx.seeds, gy.seeds, pub.2.2.2.2.2.1]⟩ + · have L := RV.lrel hp hTG h + exact ⟨WP.mono (rej4At_ok C.rej4 L.1 hrej) fun _ h' => ⟨_, h'.1⟩, + WP.mono (rej4At_ok C.rej4 L.2.1 hrej) fun _ h' => ⟨_, h'.1⟩⟩ + +theorem IA.next {p : Params} {r : Nat} {x y : State} (h : RV p (IA p r p.ℓ) x y) : RV p (IA p (r + 1) 0) x y := by + obtain ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁⟩, ⟨h₂, hh₂, s₂⟩⟩ := h + exact ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁.next⟩, ⟨h₂, hh₂, s₂.next⟩⟩ + theorem aRow_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r : Nat} (hr : r < p.k) : RelCT isa (RV p (IA p r 0)) (aRow P p r) (RV p (IA p (r + 1) 0)) := by + have hkl := kl_le p hp + have hg := gChk_all p hp r hr + have hl : p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by + have : ∀ p ∈ params, p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by decide + exact this p hp unfold aRow - refine RelCT.mono (seqR_tr (R := fun e => RV p (IA p r (e - 8 * r))) p.ℓ (8 * r) fun e he he' => ?_) - (fun x y h => by rwa [Nat.sub_self]) fun x y h => ?_ - · have := aOne_tr C hp hr (c := e - 8 * r) (by omega) - rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this - exact this - · rw [show 8 * r + p.ℓ - 8 * r = p.ℓ by omega] at h - obtain ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁⟩, ⟨h₂, hh₂, s₂⟩⟩ := h - exact ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁.next⟩, ⟨h₂, hh₂, s₂.next⟩⟩ + refine RelCT.seq (aGrp_tr C hp hr hg.1) ?_ + by_cases h7 : p.ℓ = 7 + · rw [ite_eq_left h7] + refine RelCT.mono (aGrp_tr C hp hr (hg.2 h7)) (fun x y h => by rwa [Nat.zero_add] at h) fun x y h => ?_ + rw [show 3 + 4 = p.ℓ by omega] at h + exact IA.next h + · rw [ite_eq_right h7] + refine RelCT.mono (seqR_tr (R := fun e => RV p (IA p r (e - 8 * r))) (p.ℓ - 4) (8 * r + 4) fun e he he' => ?_) + (fun x y h => by rw [show 8 * r + 4 - 8 * r = 4 by omega]; rwa [Nat.zero_add] at h) fun x y h => ?_ + · have := aOne_tr C hp hr (c := e - 8 * r) (by omega) + rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this + exact this + · rw [show 8 * r + 4 + (p.ℓ - 4) - 8 * r = p.ℓ by omega] at h + exact IA.next h theorem ballStage_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) : RelCT isa (RV p (IA p p.k 0)) (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC) (RV p (I4 p)) := by @@ -94,6 +171,22 @@ theorem ballStage_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params exact ⟨WP.mono (ballAt_ok C.ball L.1 c6 c5) fun _ h' => ⟨_, h'.1⟩, WP.mono (ballAt_ok C.ball L.2.1 c6 c5) fun _ h' => ⟨_, h'.1⟩⟩ +/-- While copying `ρ`. -/ +def IRho (p : Params) (j : Nat) (σ s : State) : Prop := ∃ h, HN p σ h ∧ RhoS p h j σ s + +theorem copyK_tr {p : Params} (hp : p ∈ params) {j : Nat} (hj : j < 4) : + RelCT isa (RV p (IRho p j)) (copy (sc (oSB4 + 34 * j)) (.rbp, 0) 32) (RV p (IRho p (j + 1))) := by + have hc := rChk_all p hp j hj + simp only [rChk, Bool.and_eq_true] at hc + have hS := layOk p hp + have hsd' := sepB_spec hc.1.1.1.1 + have hok : ∀ a ∈ copyArgs (sc (oSB4 + 34 * j)) (.rbp, 0) 32, a.2.Ok ∧ a.1 ∈ argRegs := by + simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] + exact ⟨⟨ptr_ok hS hsd'.2.1, by decide⟩, ⟨ptr_ok hS hsd'.1, by decide⟩, ⟨show 32 < 2 ^ 31 by decide, by decide⟩⟩ + exact relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (copyK_ok hp hv hj hs) fun _ h' => ⟨h, hh, h'⟩) + (copy_tr hok (show Reg.rbx ∈ bases by decide) (by decide) fun x y h => + (RV.lrel hp (fun _ _ h => let ⟨_, _, hs⟩ := h; hs.t) h).2.2) + theorem samples_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) : RelCT isa (RV p (Is p)) (samples P p) (RV p (I4 p)) := by have hc := sChk_all p hp @@ -104,11 +197,16 @@ theorem samples_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) have hok : ∀ a ∈ copyArgs (sc oSB) (.rbp, 0) 32, a.2.Ok ∧ a.1 ∈ argRegs := by simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] exact ⟨⟨ptr_ok hS hsd'.2.1, by decide⟩, ⟨ptr_ok hS hsd'.1, by decide⟩, ⟨show 32 < 2 ^ 31 by decide, by decide⟩⟩ - unfold samples - refine RelCT.seq (relInv (I' := IA p 0 0) + unfold samples rhos + refine RelCT.seq (RelCT.seq (relInv (I' := IRho p 0) (fun σ s hv ⟨h, hh, hs, h15⟩ => WP.mono (copyRho_ok hp hv hs h15) fun _ h' => ⟨h, hh, h'⟩) (copy_tr hok (by decide) (by decide) fun x y h => - (RV.lrel hp (fun _ _ h => let ⟨_, _, hs, _⟩ := h; hs.t) h).2.2)) (RelCT.seq ?_ (ballStage_tr C hp)) + (RV.lrel hp (fun _ _ h => let ⟨_, _, hs, _⟩ := h; hs.t) h).2.2)) + (RelCT.seq (copyK_tr hp (j := 0) (by decide)) (RelCT.seq (copyK_tr hp (j := 1) (by decide)) + (RelCT.seq (copyK_tr hp (j := 2) (by decide)) (RelCT.mono (copyK_tr hp (j := 3) (by decide)) (fun _ _ h => h) + (Q' := RV p (IA p 0 0)) fun x y h => ?_))))) (RelCT.seq ?_ (ballStage_tr C hp)) + · obtain ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, r₁⟩, ⟨h₂, hh₂, r₂⟩⟩ := h + exact ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, r₁.s3⟩, ⟨h₂, hh₂, r₂.s3⟩⟩ have := seqR_tr (R := fun r => RV p (IA p r 0)) p.k 0 fun r _ hr => aRow_tr C hp (by omega) rwa [Nat.zero_add] at this diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTStages.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTStages.lean index 9933336e8..9647e78c8 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTStages.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTStages.lean @@ -43,7 +43,12 @@ theorem and15_post (x : State) : WP isa (.block and15) x fun x' => ∃ W, PostB theorem sampledTail_tr {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) : RelCT isa (fun x y => SameB x y) (.seq (.block and15) (mask a)) fun _ _ => True := RelCT.seq (RelCT.sameB and15_tr (fun x y _ => ⟨and15_post x, and15_post y⟩) fun _ _ h => h) - (mask_tr hok hb fun _ _ h => h) + (mask_tr hok hb (N := 256) (by decide) fun _ _ h => h) + +theorem sampledTail4_tr {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) : + RelCT isa (fun x y => SameB x y) (.seq (.block and15) (mask a 1024)) fun _ _ => True := + RelCT.seq (RelCT.sameB and15_tr (fun x y _ => ⟨and15_post x, and15_post y⟩) fun _ _ h => h) + (mask_tr hok hb (N := 1024) (by decide) fun _ _ h => h) theorem flag_ne {P : Prop} [Decidable P] (h : (flag P).setWidth 32 ≠ 0) : P := by by_contra hn diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Call.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Call.lean index e1e56ef2b..eb9c0af10 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Call.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Call.lean @@ -8,7 +8,7 @@ import VerifiedGarbage.Proof.Framework.Contract Untrusted: everything here is checked by Lean. A call of verified code, with the moves of its arguments before it (`callAt_ok`), leaves the permissions and the callee-saved registers as they were, and changes memory -only within the buffers it writes and the 24 bytes of stack below `rsp` +only within the buffers it writes and the 32 bytes of stack below `rsp` (`Post`); two runs whose arguments agree and whose callee's public data agree leak the same (`callAt_tr`). A callee may be verified against its contract with any stack up to 16 bytes: its precondition follows from the @@ -61,7 +61,7 @@ abbrev Args (as : List (Reg × Arg)) (s s1 : State) : Prop := theorem callAt_ok {n : String} {c : Prog isa} {k : Contract isa} (hv : ∀ s, k.pre s → ∃ t s', Exec isa c s t s' ∧ abiPreserved s s' ∧ k.post s s') - (hsp : NoSp c) (hd : c.depth ≤ 2) {as : List (Reg × Arg)} (hok : ∀ a ∈ as, a.2.Ok ∧ a.1 ∈ argRegs) + (hsp : NoSp c) (hd : c.depth ≤ 3) {as : List (Reg × Arg)} (hok : ∀ a ∈ as, a.2.Ok ∧ a.1 ∈ argRegs) (hnd : (as.map (·.1)).Nodup) {s : State} {rd wr : List Region} (hpre : ∀ s1, Args as s s1 → k.pre (s1.callEntry.withRegions rd wr)) (hc : Covers (rd ++ wr) (s.rd ++ s.wr)) (hw : Covers wr s.wr) : diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CallSample.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CallSample.lean index 67939bf56..7d71683f7 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CallSample.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CallSample.lean @@ -96,6 +96,96 @@ theorem rejNttAt_tr {P : Prims} (C : CalleeOk P.rejNtt (rejNTTContract X86_64.ab rw [Lx.wbytesAt c4, Ly.wbytesAt c4, eb] exact ⟨by rw [e.2], rfl, e.pa (ptr_bs hS c4), e.pa (ptr_bs hS c5), e.pa (ptr_bs hS c6)⟩ +/-! ## `RejNTTPoly` four times -/ + +def rej4Chk (bs wbs : List (Reg × Nat)) (a w : Ptr) : Bool := + sepB bs (sc oSB4) 136 a 4096 && sepB bs (sc oSB4) 136 w 8192 && sepB bs a 4096 w 8192 && + inB bs (sc oSB4) 136 && inB bs a 4096 && inB bs w 8192 && inB wbs a 4096 && inB wbs w 8192 + +abbrev rej4Args (a w : Ptr) : List (Reg × Arg) := [(.rdi, .ptr (sc oSB4)), (.rsi, .ptr a), (.rdx, .ptr w)] + +theorem rej4_args {bs wbs : List (Reg × Nat)} (L : LayOk bs) {a w : Ptr} (hc : rej4Chk bs wbs a w = true) : + ∀ x ∈ rej4Args a w, x.2.Ok ∧ x.1 ∈ argRegs := by + simp only [rej4Chk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨⟨⟨_, c4⟩, c5⟩, c6⟩, _⟩, _⟩ := hc + simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] + exact ⟨⟨ptr_ok L c4, by decide⟩, ⟨ptr_ok L c5, by decide⟩, ⟨ptr_ok L c6, by decide⟩⟩ + +section +variable {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a w : Ptr} + (hc : rej4Chk (rbs ++ wbs) wbs a w = true) +include L hc + +theorem rej4_cov : Covers ([⟨pa s (sc oSB4), 136⟩] ++ [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩]) (s.rd ++ s.wr) ∧ + Covers [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩] s.wr := by + simp only [rej4Chk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨⟨⟨_, c4⟩, _⟩, _⟩, c7⟩, c8⟩ := hc + exact ⟨covers_append (L.cR c4) (covers_wr (covers_cons (L.cW c7) (L.cW c8))), covers_cons (L.cW c7) (L.cW c8)⟩ + +theorem rej4_pre {s1 : State} (h1 : Args (rej4Args a w) s s1) : + (rejNTT4Contract X86_64.abi 24).pre + (s1.callEntry.withRegions [⟨pa s (sc oSB4), 136⟩] [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩]) := by + simp only [rej4Chk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨⟨⟨⟨⟨c1, c2⟩, c3⟩, c4⟩, c5⟩, c6⟩, _⟩, _⟩ := hc + have g1 : s1.gpr .rdi = pa s (sc oSB4) := h1.r0 + have g2 : s1.gpr .rsi = pa s a := h1.r1 + have g3 : s1.gpr .rdx = pa s w := h1.r2 + sig_pre [rejNTT4Contract, rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] + rw [g1, g2, g3, h1.rsp] + exact ⟨L.sp24, rfl, rfl, L.disj c1, L.disj c2, L.disj c3, L.ret8 c4, L.ret8 c5, L.ret8 c6, L.stk24 c4, + L.stk24 c5, L.stk24 c6, L.nwp c4, L.nwp c5, L.nwp c6⟩ + +end + +theorem Lay.wseed4 {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) + (h : inB (rbs ++ wbs) (sc oSB4) 136 = true) (v : BitVec 64) {k : Nat} (hk : k < 4) : + seed4 (s.mem.writeW (s.gpr .rsp - 8) v) (pa s (sc oSB4)) k = seed4 s.mem (pa s (sc oSB4)) k := by + unfold seed4 + refine Proof.MlKem.bytesAt_congr fun i hi => ?_ + rw [BitVec.add_assoc, ← BitVec.ofNat_add] + exact L.wbytes h v (34 * k + i) (by omega) + +theorem rej4At_ok {P : Prims} (C : CalleeOk P.rej4 (rejNTT4Contract X86_64.abi 24)) {rbs wbs : List (Reg × Nat)} + {s : State} (L : Lay rbs wbs s) {a w : Ptr} (hc : rej4Chk (rbs ++ wbs) wbs a w = true) : + WP isa (rej4At P a w) s fun s' => PPostB s s' [(a, 4096), (w, 8192)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ + (res s' = 1 → ∀ k < 4, Reduced s'.mem (poly4 (pa s a) k)) ∧ + ((res s' = 1 ∧ ∀ k < 4, ∃ b : Bounds, rejNTTPoly b.rejNTT (seed4 s.mem (pa s (sc oSB4)) k) = + some (polyAt s'.mem (poly4 (pa s a) k))) ∨ + (res s' = 0 ∧ ∃ k < 4, rejNTTPoly minBounds.rejNTT (seed4 s.mem (pa s (sc oSB4)) k) = none)) := by + refine WP.mono (callAt_ok C.correct C.nosp C.depth (rej4_args L.ok hc) + (by simp only [List.map_cons, List.map_nil]; decide) + (fun s1 h1 => rej4_pre L hc h1) (rej4_cov L hc).1 (rej4_cov L hc).2) + fun s' ⟨hP, s1, h1, s₂, hm, hg, hq⟩ => ⟨hP.b, hP.cs .r15 (by decide), ?_⟩ + simp only [rej4Chk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨⟨⟨_, c4⟩, _⟩, _⟩, _⟩, _⟩ := hc + sig_post [rejNTT4Contract, rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] at hq + rw [h1.r0, h1.r1, hm, h1.rsp, h1.1.2, hg _ (by decide)] at hq + simp only [Arg.val] at hq + obtain ⟨hr, ho⟩ := hq + refine ⟨hr, ?_⟩ + rcases ho with ⟨h1', hb⟩ | ⟨h0, k, hk, hn⟩ + · exact .inl ⟨h1', fun k hk => by rw [← L.wseed4 c4 _ hk]; exact hb k hk⟩ + · exact .inr ⟨h0, k, hk, by rw [← L.wseed4 c4 _ hk]; exact hn⟩ + +theorem rej4At_tr {P : Prims} (C : CalleeOk P.rej4 (rejNTT4Contract X86_64.abi 24)) {rbs wbs : List (Reg × Nat)} + (hS : LayOk (rbs ++ wbs)) {a w : Ptr} (hc : rej4Chk (rbs ++ wbs) wbs a w = true) {Q : State → State → Prop} + (hQ : ∀ x y, Q x y → Lay rbs wbs x ∧ Lay rbs wbs y ∧ SameB x y ∧ + bytesAt x.mem (pa x (sc oSB4)) 136 = bytesAt y.mem (pa y (sc oSB4)) 136) : + RelCT isa Q (rej4At P a w) fun _ _ => True := by + refine callAt_tr C.correct C.ct (rej4_args hS hc) (by simp only [List.map_cons, List.map_nil]; decide) ?_ + intro x y x1 y1 hp h1 h2 + obtain ⟨Lx, Ly, e, eb⟩ := hQ x y hp + refine ⟨_, _, _, _, rej4_pre Lx hc h1, rej4_pre Ly hc h2, ?_, (rej4_cov Lx hc).1, (rej4_cov Lx hc).2, + (rej4_cov Ly hc).1, (rej4_cov Ly hc).2, e.2⟩ + have hc' := hc + simp only [rej4Chk, Bool.and_eq_true] at hc' + obtain ⟨⟨⟨⟨⟨_, c4⟩, c5⟩, c6⟩, _⟩, _⟩ := hc' + sig_pub [rejNTT4Contract, rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] + rw [h1.r0, h1.r1, h1.r2, h2.r0, h2.r1, h2.r2, h1.1.2, h2.1.2, h1.rsp, h2.rsp] + simp only [Arg.val] + rw [Lx.wbytesAt c4, Ly.wbytesAt c4, eb] + exact ⟨by rw [e.2], rfl, e.pa (ptr_bs hS c4), e.pa (ptr_bs hS c5), e.pa (ptr_bs hS c6)⟩ + /-! ## `SampleInBall` -/ def ballChk (bs wbs : List (Reg × Nat)) (ct : Ptr) (len : Nat) (c : Ptr) : Bool := diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean index 430f0bb51..fde6d0c05 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean @@ -11,8 +11,9 @@ needs on its entry, from the layout of the caller: that its buffers are apart from its return address and the 16 bytes of stack below it, and from each other, and that they read on entry as they did before the call (`Ent`). And what the callees must be (`CalleeOk`): correct and constant -time under their contracts with 16 bytes of stack, not writing the stack -pointer, calling at most two deep, and never loading MXCSR. +time under their contracts with 16 bytes of stack (24 for +`vg_mldsa_rej_ntt_poly4`), not writing the stack pointer, calling at most +three deep, and never loading MXCSR. -/ namespace VG.Proof.MlDsa.X86_64.Verify @@ -25,14 +26,14 @@ open VG.Spec.Sha3 (bytesAt) /-! ## Callees -/ /-- A callee: correct and constant time under the contract `k` (a shared -contract with 16 bytes of stack), not writing `rsp`, calling at most two -deep, loading MXCSR only to restore it (`ctlOk`), and never writing the -stack pointer (which its callers' artifacts check). -/ +contract), not writing `rsp`, calling at most three deep, loading MXCSR only +to restore it (`ctlOk`), and never writing the stack pointer (which its +callers' artifacts check). -/ structure CalleeOk (c : Prog isa) (k : Contract isa) : Prop where correct : ∀ s, k.pre s → ∃ t s', Exec isa c s t s' ∧ abiPreserved s s' ∧ k.post s s' ct : ConstantTime isa k.pre k.pub c nosp : NoSp c - depth : c.depth ≤ 2 + depth : c.depth ≤ 3 ctl : ctlOk c = true spSafe : c.all (fun i => !isa.writesSp i) = true @@ -41,7 +42,7 @@ theorem CalleeOk.of_verified {c : Prog isa} {sig : Sig} {pre : Curry (sig.words {post : sig.Post X86_64.abi.ptrBits} {wa : Bool} {leak : Option (Curry (sig.words X86_64.abi.ptrBits) (Mem → List Nat))} {n : Nat} (h : Verified X86_64.target c (sig.contract X86_64.abi pre post wa n leak)) (hn : n ≤ 16) - (hsp : NoSp c) (hd : c.depth ≤ 2) (hmx : ctlOk c = true) + (hsp : NoSp c) (hd : c.depth ≤ 3) (hmx : ctlOk c = true) (hss : c.all (fun i => !isa.writesSp i) = true) : CalleeOk c (sig.contract X86_64.abi pre post wa 16 leak) := ⟨fun s hs => h.1 s (pre_stack hn hs), @@ -59,7 +60,7 @@ variable {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) include L theorem Lay.sp16 : 16 ≤ (s.gpr .rsp - 8).toNat := by - have := L.sp24 + have := L.sp32 rw [BitVec.toNat_sub, show (8 : BitVec 64).toNat = 8 from rfl] omega @@ -68,8 +69,8 @@ theorem Lay.ret8 {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : refine (L.stkD h).sub_left ?_ intro x hx simp only [Region.Contains] at hx ⊢ - rw [show x - (s.gpr .rsp - BitVec.ofNat 64 24) = (x - (s.gpr .rsp - 8)) + 16 by bv_omega, BitVec.toNat_add] - have : (16 : BitVec 64).toNat = 16 := rfl + rw [show x - (s.gpr .rsp - BitVec.ofNat 64 32) = (x - (s.gpr .rsp - 8)) + 24 by bv_omega, BitVec.toNat_add] + have : (24 : BitVec 64).toNat = 24 := rfl omega theorem Lay.stk16 {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : @@ -77,7 +78,21 @@ theorem Lay.stk16 {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : refine (L.stkD h).sub_left ?_ intro x hx simp only [Region.Contains] at hx ⊢ - rw [show x - (s.gpr .rsp - BitVec.ofNat 64 24) = x - (s.gpr .rsp - 8 - 16) by bv_omega] + rw [show x - (s.gpr .rsp - BitVec.ofNat 64 32) = (x - (s.gpr .rsp - 8 - 16)) + 8 by bv_omega, BitVec.toNat_add] + have : (8 : BitVec 64).toNat = 8 := rfl + omega + +theorem Lay.sp24 : 24 ≤ (s.gpr .rsp - 8).toNat := by + have := L.sp32 + rw [BitVec.toNat_sub, show (8 : BitVec 64).toNat = 8 from rfl] + omega + +theorem Lay.stk24 {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : + Region.Disjoint ⟨s.gpr .rsp - 8 - 24, 24⟩ ⟨pa s p, l⟩ := by + refine (L.stkD h).sub_left ?_ + intro x hx + simp only [Region.Contains] at hx ⊢ + rw [show x - (s.gpr .rsp - BitVec.ofNat 64 32) = x - (s.gpr .rsp - 8 - 24) by bv_omega] omega theorem Lay.wbytes {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) (v : BitVec 64) : diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Flag.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Flag.lean index 10d54497f..fe946ecd7 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Flag.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Flag.lean @@ -112,4 +112,57 @@ theorem sampled_ok {call : Prog isa} {rbs wbs : List (Reg × Nat)} {s : State} ( · rw [h1] at h0; cases h0 · exact hn +theorem coeffAt_poly4 (m : Mem) (p : Addr) (k j : Nat) : coeffAt m (poly4 p k) j = coeffAt m p (256 * k + j) := by + unfold coeffAt poly4 + rw [BitVec.add_assoc, ← BitVec.ofNat_add, show 1024 * k + 4 * j = 4 * (256 * k + j) by omega] + +/-- `sampled_ok` for a call that samples the four polynomials from `a`, each +from `F k`. -/ +theorem sampled4_ok {call : Prog isa} {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a : Ptr} + (hi : inB (rbs ++ wbs) a 4096 = true) (hw : inB wbs a 4096 = true) {p : Prop} [Decidable p] + (h15 : s.gpr .r15 = flag p) {ws : List (Ptr × Nat)} (haw : (a, 4096) ∈ ws) + {F : Nat → Bounds → Option Poly} + (hcall : WP isa call s fun s' => PPostB s s' ws ∧ s'.gpr .r15 = s.gpr .r15 ∧ + (res s' = 1 → ∀ k < 4, Reduced s'.mem (poly4 (pa s a) k)) ∧ + ((res s' = 1 ∧ ∀ k < 4, ∃ b, F k b = some (polyAt s'.mem (poly4 (pa s a) k))) ∨ + (res s' = 0 ∧ ∃ k < 4, F k minBounds = none))) : + WP isa (sampled4 call a) s fun s' => PPostB s s' ws ∧ (∀ k < 4, Reduced s'.mem (poly4 (pa s a) k)) ∧ + ∃ r : BitVec 32, (r = 1 ∨ r = 0) ∧ s'.gpr .r15 = flag (p ∧ r = 1) ∧ + (r = 1 → ∀ k < 4, ∃ b, F k b = some (polyAt s'.mem (poly4 (pa s a) k))) ∧ + (r = 0 → ∃ k < 4, F k minBounds = none) := by + unfold sampled4 + refine WP.seq (WP.mono hcall fun s₁ ⟨hP₁, h15₁, hr₁, ho₁⟩ => ?_) + have L₁ := L.post hP₁ + have hab := ptr_bs L.ok hi + have e₁ : pa s₁ a = pa s a := hP₁.pa hab + have hr : res s₁ = 1 ∨ res s₁ = 0 := by rcases ho₁ with ⟨h, _⟩ | ⟨h, _⟩ <;> [exact .inl h; exact .inr h] + refine WP.seq (WP.mono (and15_ok s₁) fun s₂ ⟨⟨h15₂, hm₂⟩, k₂⟩ => ?_) + have hP₂ : PPostB s₁ s₂ [] := postB_of_keep k₂ (by decide) (by rw [hm₂]; exact Frame.refl _ _) + have L₂ := L₁.post hP₂ + have e₂ : pa s₂ a = pa s₁ a := hP₂.pa hab + have hax : (s₂.gpr .rax).setWidth 32 = res s₁ := by rw [k₂.gpr (by decide)] + refine WP.mono (maskN_ok L₂ (N := 1024) (by decide) (by decide) hi hw) fun s₃ ⟨hP₃, h15₃, hc₃⟩ => ?_ + have hsub : ∀ w ∈ [(a, 4 * 1024)], w ∈ ws := by simp only [List.mem_singleton, forall_eq]; exact haw + have hcs : ∀ w ∈ [(a, 4 * 1024)], w.1.1 ∈ bases := by simp only [List.mem_singleton, forall_eq]; exact hab + have hP₁₂ : PPostB s s₂ ws := PPostB.trans hP₁ hP₂ (fun _ h => absurd h List.not_mem_nil) (fun w hw => hw) + (fun _ h => absurd h List.not_mem_nil) + have hP : PPostB s s₃ ws := PPostB.trans hP₁₂ hP₃ hcs (fun w hw => hw) hsub + rw [e₂, e₁, hax, hm₂] at hc₃ + have hc : ∀ k < 4, ∀ j < n, coeffAt s₃.mem (poly4 (pa s a) k) j = + coeffAt s₁.mem (poly4 (pa s a) k) j &&& (0 - res s₁) := fun k hk j hj => by + rw [coeffAt_poly4, coeffAt_poly4, hc₃ _ (by simp only [n] at hj; omega)] + refine ⟨hP, fun k hk => ?_, res s₁, hr, by rw [h15₃, h15₂, h15₁, h15, and_flag hr], fun h1 k hk => ?_, + fun h0 => ?_⟩ + · rcases hr with h1 | h0 + · exact reduced_of_coeffAt (fun i hi => by rw [hc k hk i hi, mask_one _ h1 i hi]) (hr₁ h1 k hk) + · exact (Proof.MlDsa.Verify.polyIs_zero fun i hi => by rw [hc k hk i hi, mask_zero _ h0 i hi]).1 + · rcases ho₁ with ⟨_, hb⟩ | ⟨h0, _⟩ + · obtain ⟨b, hb⟩ := hb k hk + refine ⟨b, ?_⟩ + rw [hb, polyAt_eq_of_coeffAt fun i hi => by rw [hc k hk i hi, mask_one _ h1 i hi]] + · rw [h1] at h0; cases h0 + · rcases ho₁ with ⟨h1, _⟩ | ⟨_, hn⟩ + · rw [h1] at h0; cases h0 + · exact hn + end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Hash.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Hash.lean index dd764ad70..380012376 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Hash.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Hash.lean @@ -44,8 +44,8 @@ variable {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) include L theorem kChk_spec (h : kChk (rbs ++ wbs) wbs = true) : - Region.Disjoint ⟨pa s (sc 0), 200⟩ ⟨pa s (sc 200), 640⟩ ∧ (below (s.gpr .rsp) 24).Disjoint ⟨pa s (sc 0), 200⟩ ∧ - (below (s.gpr .rsp) 24).Disjoint ⟨pa s (sc 200), 640⟩ ∧ Covers [⟨pa s (sc 0), 200⟩] s.wr ∧ + Region.Disjoint ⟨pa s (sc 0), 200⟩ ⟨pa s (sc 200), 640⟩ ∧ (below (s.gpr .rsp) 32).Disjoint ⟨pa s (sc 0), 200⟩ ∧ + (below (s.gpr .rsp) 32).Disjoint ⟨pa s (sc 200), 640⟩ ∧ Covers [⟨pa s (sc 0), 200⟩] s.wr ∧ Covers [⟨pa s (sc 200), 640⟩] s.wr := by simp only [kChk, Bool.and_eq_true] at h obtain ⟨⟨⟨⟨c1, c2⟩, c3⟩, c4⟩, c5⟩ := h @@ -98,7 +98,7 @@ theorem kabs_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) (h refine WP.seq (WP.mono (glue_ok' ha (by simp only [List.map_cons, List.map_nil]; decide) s) fun s1 (h1 : Args (kabsArgs src len 136 pos) s s1) => ?_) have hsp : s1.gpr .rsp = s.gpr .rsp := h1.rsp - have kk : ∀ {R : Region}, (below (s.gpr .rsp) 24).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := + have kk : ∀ {R : Region}, (below (s.gpr .rsp) 32).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := fun h => by rw [hsp]; exact h.sub_left (below_sub (by omega) (by omega)) refine absorb_call ⟨h1.r0, h1.r1, h1.r2, h1.r3, h1.r4, h1.r5, by decide, hpos, by omega, d1, (L.disj p1), (L.disj p2), kk k1, kk (L.stkD p3), kk k2⟩ @@ -113,7 +113,7 @@ theorem kabs_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) (h fun msg hm hpo => ?_⟩ · have f1 : Frame ([⟨pa s (sc 0), 200⟩, ⟨pa s (sc 200), 640⟩] ++ [below (s.gpr .rsp) 16]) s.mem s'.mem := by rw [← h1.1.2, ← hsp]; exact hf - exact Frame.below_mono (a := 16) (b := 24) f1 (by omega) (by omega) + exact Frame.below_mono (a := 16) (b := 32) f1 (by omega) (by omega) · rw [← h1.1.2] at hm ⊢ exact hR msg hm hpo @@ -125,7 +125,7 @@ theorem postB_call {s s1 s' : State} {as : List (Reg × Arg)} (h1 : Args as s s1 (hf : Frame (W ++ [below (s.gpr .rsp) 16]) s.mem s'.mem) : PostB s s' W := by have hsp : s1.gpr .rsp = s.gpr .rsp := h1.rsp refine ⟨hrd.trans h1.2.2.1, hwr.trans h1.2.2.2, fun r hr => ?_, by rw [hcs _ (by decide), hsp], - Frame.below_mono (a := 16) (b := 24) hf (by omega) (by omega)⟩ + Frame.below_mono (a := 16) (b := 32) hf (by omega) (by omega)⟩ simp only [bases, List.mem_cons, List.not_mem_nil, or_false] at hr rw [hcs r (by rcases hr with rfl | rfl | rfl | rfl <;> decide), h1.2.gpr (by rcases hr with rfl | rfl | rfl | rfl <;> decide)] @@ -148,7 +148,7 @@ theorem kpad_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) (h refine WP.seq (WP.mono (glue_ok' ha (by simp only [List.map_cons, List.map_nil]; decide) s) fun s1 (h1 : Args (kpadArgs 136 pos 0x1f) s s1) => ?_) have hsp : s1.gpr .rsp = s.gpr .rsp := h1.rsp - have kk : ∀ {R : Region}, (below (s.gpr .rsp) 24).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := + have kk : ∀ {R : Region}, (below (s.gpr .rsp) 32).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := fun h => by rw [hsp]; exact h.sub_left (below_sub (by omega) (by omega)) refine pad_call ⟨h1.r0, h1.r1, h1.r2, h1.r4, by decide, hpos, d1, kk k1, kk k2⟩ (by rw [h1.2.2.1, h1.2.2.2]; exact covers_append covers_nil (covers_wr (covers_cons w1 w2))) @@ -185,7 +185,7 @@ theorem ksqz_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) (h refine WP.seq (WP.mono (glue_ok' ha (by simp only [List.map_cons, List.map_nil]; decide) s) fun s1 (h1 : Args (ksqzArgs 136 out len) s s1) => ?_) have hsp : s1.gpr .rsp = s.gpr .rsp := h1.rsp - have kk : ∀ {R : Region}, (below (s.gpr .rsp) 24).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := + have kk : ∀ {R : Region}, (below (s.gpr .rsp) 32).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := fun h => by rw [hsp]; exact h.sub_left (below_sub (by omega) (by omega)) refine squeeze_call ⟨h1.r0, h1.r1, h1.r2, h1.r3, h1.r4, h1.r5, by decide, by decide, by omega, (L.disj o1).symm, d1, L.disj o2, kk k1, kk (L.stkD o3), kk k2⟩ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/HashCT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/HashCT.lean index afbc83ecb..9d3c51ff0 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/HashCT.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/HashCT.lean @@ -51,7 +51,7 @@ theorem k_in {bs wbs : List (Reg × Nat)} (hk : kChk bs wbs = true) : inB bs (sc 0) 200 = true ∧ inB bs (sc 200) 640 = true := by simp only [kChk, Bool.and_eq_true] at hk; exact ⟨hk.1.1.1.2, hk.1.1.2⟩ -theorem kk16 {s s1 : State} (hsp : s1.gpr .rsp = s.gpr .rsp) {R : Region} (h : (below (s.gpr .rsp) 24).Disjoint R) : +theorem kk16 {s s1 : State} (hsp : s1.gpr .rsp = s.gpr .rsp) {R : Region} (h : (below (s.gpr .rsp) 32).Disjoint R) : (below (s1.gpr .rsp) 16).Disjoint R := by rw [hsp]; exact h.sub_left (below_sub (by omega) (by omega)) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean index 852e3e7a1..758076ee2 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean @@ -21,7 +21,7 @@ open VG.Spec.MlDsa /-- The primitives, each empty. -/ def P0 : Prims := ⟨.block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], - .block [], .block [], .block [], .block [], ""⟩ + .block [], .block [], .block [], .block [], .block [], ""⟩ /-- `q` holds of every instruction of the primitives `P`. -/ structure PrimsQ (q : Instr → Bool) (P : Prims) : Prop where @@ -38,6 +38,7 @@ structure PrimsQ (q : Instr → Bool) (P : Prims) : Prop where unpackT1 : P.unpackT1.allInstrs q = true hintUnpack : P.hintUnpack.allInstrs q = true normLt : P.normLt.allInstrs q = true + rej4 : P.rej4.allInstrs q = true /-- `q` holds of every instruction of `c` exactly when it does of `c'`. -/ def SameQ (q : Instr → Bool) (c c' : Prog isa) : Prop := c.allInstrs q = c'.allInstrs q @@ -68,6 +69,9 @@ theorem SameQ.ifOk {c c' : Prog isa} (h : SameQ q c c') : SameQ q (ifOk c) (ifOk theorem SameQ.sampled {c c' : Prog isa} (h : SameQ q c c') (a : Ptr) : SameQ q (sampled c a) (sampled c' a) := h.seq rfl +theorem SameQ.sampled4 {c c' : Prog isa} (h : SameQ q c c') (a : Ptr) : SameQ q (sampled4 c a) (sampled4 c' a) := + h.seq rfl + variable {P : Prims} (hP : PrimsQ q P) (p : Params) include hP @@ -80,7 +84,14 @@ omit p in theorem aOne_q (e : Nat) : SameQ q (aOne P e) (aOne P0 e) := SameQ.seq rfl (SameQ.sampled (SameQ.call hP.rejNtt _) _) -theorem aRow_q (r : Nat) : SameQ q (aRow P p r) (aRow P0 p r) := SameQ.seqR (aOne_q hP) _ _ +theorem aGrp_q (r c : Nat) : SameQ q (aGrp P p r c) (aGrp P0 p r c) := + SameQ.seq rfl (SameQ.seq rfl (SameQ.seq rfl (SameQ.seq rfl (SameQ.sampled4 (SameQ.call hP.rej4 _) _)))) + +theorem aRow_q (r : Nat) : SameQ q (aRow P p r) (aRow P0 p r) := by + refine (aGrp_q hP p r 0).seq ?_ + by_cases h : p.ℓ = 7 + · rw [ite_eq_left h, ite_eq_left h]; exact aGrp_q hP p r 3 + · rw [ite_eq_right h, ite_eq_right h]; exact SameQ.seqR (aOne_q hP) _ _ theorem samples_q : SameQ q (samples P p) (samples P0 p) := SameQ.seq rfl ((SameQ.seqR (aRow_q hP p) _ _).seq (SameQ.sampled (SameQ.call hP.ball _) _)) @@ -123,6 +134,7 @@ structure PrimsC (P : Prims) : Prop where unpackT1 : ctlOk P.unpackT1 = true hintUnpack : ctlOk P.hintUnpack = true normLt : ctlOk P.normLt = true + rej4 : ctlOk P.rej4 = true /-- `ctlC` holds of `c` exactly when it does of `c'`. -/ def SameC (c c' : Prog isa) : Prop := ctlC c = ctlC c' @@ -148,6 +160,9 @@ theorem SameC.ifOk {c c' : Prog isa} (h : SameC c c') : SameC (ifOk c) (ifOk c') theorem SameC.sampled {c c' : Prog isa} (h : SameC c c') (a : Ptr) : SameC (sampled c a) (sampled c' a) := h.seq rfl +theorem SameC.sampled4 {c c' : Prog isa} (h : SameC c c') (a : Ptr) : SameC (sampled4 c a) (sampled4 c' a) := + h.seq rfl + section variable {P : Prims} (hP : PrimsC P) (p : Params) include hP @@ -161,8 +176,15 @@ theorem verify_c : SameC (verify P p) (verify P0 p) := by (dot r).seq ((SameC.call hP.unpackT1 _).seq ((SameC.call hP.ntt _).seq ((SameC.call hP.mul _).seq ((SameC.call hP.sub _).seq ((SameC.call hP.invNtt _).seq ((SameC.call hP.useHint _).seq (SameC.call hP.simpleBitPack _))))))) + have aGrp : ∀ r c, SameC (aGrp P p r c) (aGrp P0 p r c) := fun r c => + SameC.seq rfl (SameC.seq rfl (SameC.seq rfl (SameC.seq rfl (SameC.sampled4 (SameC.call hP.rej4 _) _)))) + have aRow : ∀ r, SameC (aRow P p r) (aRow P0 p r) := fun r => by + refine (aGrp r 0).seq ?_ + by_cases h : p.ℓ = 7 + · rw [ite_eq_left h, ite_eq_left h]; exact aGrp r 3 + · rw [ite_eq_right h, ite_eq_right h]; exact SameC.seqR aOne _ _ have samples : SameC (samples P p) (samples P0 p) := - SameC.seq rfl ((SameC.seqR (fun r => SameC.seqR aOne _ _) _ _).seq + SameC.seq rfl ((SameC.seqR aRow _ _).seq (SameC.sampled (SameC.call hP.ball _) _)) have compute : SameC (compute P p) (compute P0 p) := (SameC.seqR (fun _ => SameC.call hP.ntt _) _ _).seq ((SameC.call hP.ntt _).seq @@ -183,7 +205,7 @@ include C hp theorem verify_ctl : ctlOk (verify P p) = true := ctlOk_of_ctlC ((verify_c ⟨C.ntt.ctl, C.invNtt.ctl, C.mul.ctl, C.mulAdd.ctl, C.sub.ctl, C.rejNtt.ctl, C.ball.ctl, C.useHint.ctl, C.simpleBitPack.ctl, C.bitUnpack.ctl, C.unpackT1.ctl, C.hintUnpack.ctl, - C.normLt.ctl⟩ p).trans (verify0_ctlC p hp)) + C.normLt.ctl, C.rej4.ctl⟩ p).trans (verify0_ctlC p hp)) theorem verify_spSafe : (verify P p).all (fun i => !isa.writesSp i) = true := Code.all_of_allInstrs ((verify_q ⟨Code.allInstrs_of_all C.ntt.spSafe, Code.allInstrs_of_all C.invNtt.spSafe, @@ -191,6 +213,7 @@ theorem verify_spSafe : (verify P p).all (fun i => !isa.writesSp i) = true := Code.allInstrs_of_all C.rejNtt.spSafe, Code.allInstrs_of_all C.ball.spSafe, Code.allInstrs_of_all C.useHint.spSafe, Code.allInstrs_of_all C.simpleBitPack.spSafe, Code.allInstrs_of_all C.bitUnpack.spSafe, Code.allInstrs_of_all C.unpackT1.spSafe, - Code.allInstrs_of_all C.hintUnpack.spSafe, Code.allInstrs_of_all C.normLt.spSafe⟩ p).trans (verify0_sp p hp)) + Code.allInstrs_of_all C.hintUnpack.spSafe, Code.allInstrs_of_all C.normLt.spSafe, + Code.allInstrs_of_all C.rej4.spSafe⟩ p).trans (verify0_sp p hp)) end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean index 8d87a8c4d..ef0be7ae9 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean @@ -44,6 +44,7 @@ def prims : Prims where unpackT1 := Pack.unpackT1 hintUnpack := Pack.hintBitUnpack normLt := Round.normLt + rej4 := Sample.Rej4.rejNTT4 /-- The primitives, with the polynomial arithmetic of `B`. -/ def primsWith (B : Arith.Backend) : Prims := @@ -53,6 +54,7 @@ def primsWith (B : Arith.Backend) : Prims := mul := B.mul mulAdd := B.mulAdd sub := B.sub + rej4 := B.rej4 sfx := B.sfx } /-- A function of the polynomial arithmetic satisfies what the proofs of verification need of it. -/ @@ -60,7 +62,7 @@ theorem calleeOf {sig : Sig} {pre : Curry (sig.words X86_64.abi.ptrBits) (Mem {post : sig.Post X86_64.abi.ptrBits} {wa : Bool} {c : Prog isa} (h : FnOk (fun S => sig.contract X86_64.abi pre post wa S none) c) : CalleeOk c (sig.contract X86_64.abi pre post wa 16 none) := - CalleeOk.of_verified h.ver (by decide) h.nosp h.depth h.ctl h.sp + CalleeOk.of_verified h.ver (by decide) h.nosp (Nat.le_succ_of_le h.depth) h.ctl h.sp theorem prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) where ntt := by @@ -106,11 +108,12 @@ theorem prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) where (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) (Code.all_of_allInstrs (by lit_decide)) : CalleeOk prims.normLt _) + rej4 := ⟨v.ok.rej4.ver.1, v.ok.rej4.ver.2.1, v.ok.rej4.nosp, v.ok.rej4.depth, v.ok.rej4.ctl, v.ok.rej4.sp⟩ /-- `vg_mldsa*_verify` for the parameter set `p`, calling the x86-64 primitives, with the polynomial arithmetic of `v`. -/ theorem verify_prims (v : ArithImpl) {p : Spec.MlDsa.Params} (hp : p ∈ params) : - Verified X86_64.target (verify (primsWith v.code) p) (Spec.MlDsa.verifyContract p X86_64.abi 24) := + Verified X86_64.target (verify (primsWith v.code) p) (Spec.MlDsa.verifyContract p X86_64.abi 32) := verify_verified (prims_okWith v) hp end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/PrimsOk.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/PrimsOk.lean index 5ad8ebd2e..3d87fe8e3 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/PrimsOk.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/PrimsOk.lean @@ -6,9 +6,9 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Verify.Control Untrusted: everything here is checked by Lean. `PrimsOk P`: each primitive of `P` is correct and constant time under its shared contract with at most 16 bytes of stack (`CalleeOk`, from its `Verified` proof by -`CalleeOk.of_verified`), never writes the stack pointer, calls at most two -deep and never loads MXCSR. The proofs of `vg_mldsa*_verify` hold for any -such `P`. +`CalleeOk.of_verified`; 24 for `vg_mldsa_rej_ntt_poly4`), never writes the +stack pointer, calls at most three deep and never loads MXCSR. The proofs +of `vg_mldsa*_verify` hold for any such `P`. -/ namespace VG.Proof.MlDsa.X86_64.Verify @@ -31,5 +31,6 @@ structure PrimsOk (P : Prims) : Prop where unpackT1 : CalleeOk P.unpackT1 (unpackT1Contract X86_64.abi 16) hintUnpack : CalleeOk P.hintUnpack (hintBitUnpackContract X86_64.abi 16) normLt : CalleeOk P.normLt (normLtContract X86_64.abi 16) + rej4 : CalleeOk P.rej4 (rejNTT4Contract X86_64.abi 24) end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean index 79e5d8548..63bbf0e86 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean @@ -50,6 +50,7 @@ structure S3 (p : Params) (h : List (Vector Bool n)) (r c : Nat) (σ st : State) hint : HintIs st.mem (pa st (pH 0)) p.k h z : ∀ i < p.ℓ, PolyIs st.mem (pa st (pZ i)) (toRq (vZ p (vSig p σ) i)) rho : bytesAt st.mem (pa st (sc oSB)) 32 = vRho (vPk p σ) + rho4 : ∀ k < 4, bytesAt st.mem (pa st (sc (oSB4 + 34 * k))) 32 = vRho (vPk p σ) red : ∀ r' < p.k, ∀ c' < p.ℓ, Done r c r' c' → Reduced st.mem (pa st (pA r' c')) ok : ∃ q : Bool, st.gpr .r15 = flag (q = true) ∧ (q = true → ∀ r' < p.k, ∀ c' < p.ℓ, Done r c r' c' → @@ -69,7 +70,9 @@ def aChk (p : Params) (e : Nat) : Bool := inB (vB p) (pS (20 + e)) 1024 && inB (vW p) (pS (20 + e)) 1024 && keepChk p wsB && keepChk p (wsA e) && keepB (vB p) wsB (sc oSB) 32 && keepB (vB p) (wsA e) (sc oSB) 32 && (List.range p.k).all (fun r' => (List.range p.ℓ).all fun c' => !decide (8 * r' + c' < e) || - (keepB (vB p) wsB (pS (20 + 8 * r' + c')) 1024 && keepB (vB p) (wsA e) (pS (20 + 8 * r' + c')) 1024)) + (keepB (vB p) wsB (pS (20 + 8 * r' + c')) 1024 && keepB (vB p) (wsA e) (pS (20 + 8 * r' + c')) 1024)) && + (List.range 4).all (fun k => keepB (vB p) wsB (sc (oSB4 + 34 * k)) 32 && + keepB (vB p) (wsA e) (sc (oSB4 + 34 * k)) 32) theorem aChk_all : ∀ p ∈ params, ∀ r < p.k, ∀ c < p.ℓ, aChk p (8 * r + c) = true := by decide @@ -79,20 +82,22 @@ theorem keepChk_spec {p : Params} {ws : List (Ptr × Nat)} (h : keepChk p ws = t simp only [keepChk, Bool.and_eq_true, List.all_eq_true, List.mem_range] at h exact ⟨h.1.1, h.1.2, h.2⟩ -theorem setB2_ok {p : Params} {s : State} (L : Lay (vR p) (vW p) s) (h32 : inB (vW p) (sc (oSB + 32)) 1 = true) - (h33 : inB (vW p) (sc (oSB + 33)) 1 = true) (x y : Nat) (hx : x < 256) (hy : y < 256) : - WP isa (.block (setB (sc (oSB + 32)) x ++ setB (sc (oSB + 33)) y)) s fun s' => PPostB s s' wsB ∧ +theorem setB2At_ok {p : Params} {s : State} (L : Lay (vR p) (vW p) s) (B : Nat) + (h32 : inB (vW p) (sc (B + 32)) 1 = true) + (h33 : inB (vW p) (sc (B + 33)) 1 = true) (x y : Nat) (hx : x < 256) (hy : y < 256) : + WP isa (.block (setB (sc (B + 32)) x ++ setB (sc (B + 33)) y)) s fun s' => + PPostB s s' [(sc (B + 32), 1), (sc (B + 33), 1)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ - s'.mem = (s.mem.writeW (pa s (sc oSB) + BitVec.ofNat 64 32) (BitVec.ofNat 8 x)).writeW - (pa s (sc oSB) + BitVec.ofNat 64 33) (BitVec.ofNat 8 y) := by - have e32 : pa s (sc (oSB + 32)) = pa s (sc oSB) + BitVec.ofNat 64 32 := by + s'.mem = (s.mem.writeW (pa s (sc B) + BitVec.ofNat 64 32) (BitVec.ofNat 8 x)).writeW + (pa s (sc B) + BitVec.ofNat 64 33) (BitVec.ofNat 8 y) := by + have e32 : pa s (sc (B + 32)) = pa s (sc B) + BitVec.ofNat 64 32 := by simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add] - have e33 : pa s (sc (oSB + 33)) = pa s (sc oSB) + BitVec.ofNat 64 33 := by + have e33 : pa s (sc (B + 33)) = pa s (sc B) + BitVec.ofNat 64 33 := by simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add] rw [WP.block_append_iff] - refine WP.mono (setB_ok _ x (by decide) hx s (L.inW h32)) fun s₁ ⟨hm₁, k₁⟩ => ?_ - have e₁ : pa s₁ (sc (oSB + 33)) = pa s (sc (oSB + 33)) := by simp only [pa]; rw [k₁.gpr (by decide)] - refine WP.mono (setB_ok _ y (by decide) hy s₁ (by rw [k₁.2.2, e₁]; exact L.inW h33)) fun s₂ ⟨hm₂, k₂⟩ => + refine WP.mono (setB_ok _ x (show Reg.rbx ≠ .rax by decide) hx s (L.inW h32)) fun s₁ ⟨hm₁, k₁⟩ => ?_ + have e₁ : pa s₁ (sc (B + 33)) = pa s (sc (B + 33)) := by simp only [pa]; rw [k₁.gpr (by decide)] + refine WP.mono (setB_ok _ y (show Reg.rbx ≠ .rax by decide) hy s₁ (by rw [k₁.2.2, e₁]; exact L.inW h33)) fun s₂ ⟨hm₂, k₂⟩ => ⟨postB_of_keep (k₁.trans k₂) (by decide) ?_, by rw [k₂.gpr (by decide), k₁.gpr (by decide)], ?_⟩ · rw [hm₂, hm₁, e₁] have hc : ∀ q : Ptr, (Region.mk (pa s q) 1).Contains (pa s q) 1 := fun q => Region.contains_self _ _ @@ -100,6 +105,13 @@ theorem setB2_ok {p : Params} {s : State} (L : Lay (vR p) (vW p) s) (h32 : inB ( (List.mem_cons_of_mem _ (List.mem_cons_self ..)) _ (hc _) · rw [hm₂, hm₁, e₁, e32, e33] +theorem setB2_ok {p : Params} {s : State} (L : Lay (vR p) (vW p) s) (h32 : inB (vW p) (sc (oSB + 32)) 1 = true) + (h33 : inB (vW p) (sc (oSB + 33)) 1 = true) (x y : Nat) (hx : x < 256) (hy : y < 256) : + WP isa (.block (setB (sc (oSB + 32)) x ++ setB (sc (oSB + 33)) y)) s fun s' => PPostB s s' wsB ∧ + s'.gpr .r15 = s.gpr .r15 ∧ + s'.mem = (s.mem.writeW (pa s (sc oSB) + BitVec.ofNat 64 32) (BitVec.ofNat 8 x)).writeW + (pa s (sc oSB) + BitVec.ofNat 64 33) (BitVec.ofNat 8 y) := + setB2At_ok L oSB h32 h33 x y hx hy theorem kl_le : ∀ p ∈ params, p.ℓ ≤ 7 ∧ p.k ≤ 8 := by decide @@ -114,7 +126,7 @@ theorem aOne_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ have hc8 : c < 8 := by omega simp only [aChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, Bool.or_eq_true, Bool.not_eq_true', decide_eq_false_iff_not, Nat.not_lt] at hck - obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, hwa⟩, kB⟩, kA⟩, kSB⟩, kSA⟩, kE⟩ := hck + obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, hwa⟩, kB⟩, kA⟩, kSB⟩, kSA⟩, kE⟩, k4⟩ := hck obtain ⟨tB, hB, zB⟩ := keepChk_spec kB obtain ⟨tA, hA, zA⟩ := keepChk_spec kA have L := hs.t.lay hp hv @@ -136,7 +148,9 @@ theorem aOne_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ rw [hP₂.pa (show Reg.rbx ∈ bases by decide), pA, show 20 + 8 * r + c = 20 + (8 * r + c) by omega] refine ⟨t₁.step hp hv hP₂ tA, L₁.keepHint hP₂ hA (L.keepHint hP₁ hB hs.hint), fun i hi => L₁.keepPoly hP₂ (zA i hi) (L.keepPoly hP₁ (zB i hi) (hs.z i hi)), - by rw [L₁.keepBytes hP₂ kSA, L.keepBytes hP₁ kSB, hs.rho], fun r' hr' c' hc' hd => ?_, + by rw [L₁.keepBytes hP₂ kSA, L.keepBytes hP₁ kSB, hs.rho], + fun k hk => by rw [L₁.keepBytes hP₂ (k4 k hk).2, L.keepBytes hP₁ (k4 k hk).1, hs.rho4 k hk], + fun r' hr' c' hc' hd => ?_, ⟨q && (rr == 1), ?_, fun hq r' hr' c' hc' hd => ?_, fun hq => ?_⟩⟩ · rcases done_succ.mp hd with hd | ⟨rfl, rfl⟩ · have := kE r' hr' c' hc' @@ -180,22 +194,193 @@ theorem S3.next {p : Params} {h : List (Vector Bool n)} {r : Nat} {σ s : State} have e : ∀ r' c', c' < p.ℓ → (Done r p.ℓ r' c' ↔ Done (r + 1) 0 r' c') := fun r' c' hc => by unfold Done; omega obtain ⟨q, h15, hok, hbad⟩ := hs.ok - refine ⟨hs.t, hs.hint, hs.z, hs.rho, fun r' hr' c' hc' hd => hs.red r' hr' c' hc' ((e r' c' hc').mpr hd), + refine ⟨hs.t, hs.hint, hs.z, hs.rho, hs.rho4, fun r' hr' c' hc' hd => hs.red r' hr' c' hc' ((e r' c' hc').mpr hd), q, h15, fun hq r' hr' c' hc' hd => hok hq r' hr' c' hc' ((e r' c' hc').mpr hd), fun hq => ?_⟩ obtain ⟨r', hr', c', hc', hd, hn⟩ := hbad hq exact ⟨r', hr', c', hc', (e r' c' hc').mp hd, hn⟩ +/-! ## Four entries at a time -/ + +/-- A piece writing `ws` keeps the entries done before `(r, c)` but those `ex` says, and `S3`'s other facts. -/ +def s3Chk (p : Params) (r c : Nat) (ex : Nat → Nat → Bool) (ws : List (Ptr × Nat)) : Bool := + keepChk p ws && keepB (vB p) ws (sc oSB) 32 && (List.range 4).all (fun k => keepB (vB p) ws (sc (oSB4 + 34 * k)) 32) && + (List.range p.k).all (fun r' => (List.range p.ℓ).all fun c' => + !(decide (r' < r) || (r' == r && decide (c' < c))) || ex r' c' || keepB (vB p) ws (pA r' c') 1024) + +theorem s3Chk_spec {p : Params} {r c : Nat} {ex : Nat → Nat → Bool} {ws : List (Ptr × Nat)} + (h : s3Chk p r c ex ws = true) : + keepChk p ws = true ∧ keepB (vB p) ws (sc oSB) 32 = true ∧ + (∀ k < 4, keepB (vB p) ws (sc (oSB4 + 34 * k)) 32 = true) ∧ + ∀ r' < p.k, ∀ c' < p.ℓ, Done r c r' c' → ex r' c' = false → keepB (vB p) ws (pA r' c') 1024 = true := by + simp only [s3Chk, Bool.and_eq_true] at h + obtain ⟨⟨⟨h1, h2⟩, h3⟩, h4⟩ := h + refine ⟨h1, h2, fun k hk => List.all_eq_true.mp h3 k (List.mem_range.mpr hk), fun r' hr' c' hc' hd hx => ?_⟩ + have := List.all_eq_true.mp (List.all_eq_true.mp h4 r' (List.mem_range.mpr hr')) c' (List.mem_range.mpr hc') + have hd' : (decide (r' < r) || (r' == r && decide (c' < c))) = true := by + unfold Done at hd + rcases hd with h | ⟨rfl, h⟩ <;> simp [h] + rw [hd', hx] at this + simpa using this + +/-- `S3` across a piece that writes `ws`, which `s3Chk` says keeps it. -/ +theorem S3.keep {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {r c : Nat} + {s s' : State} (hs : S3 p h r c σ s) {ws : List (Ptr × Nat)} (hP : PPostB s s' ws) + (h15 : s'.gpr .r15 = s.gpr .r15) (hc : s3Chk p r c (fun _ _ => false) ws = true) : S3 p h r c σ s' := by + obtain ⟨hk, kSB, k4, kE⟩ := s3Chk_spec hc + obtain ⟨tk, hh, zk⟩ := keepChk_spec hk + have L := hs.t.lay hp hv + obtain ⟨q, hq, hok, hbad⟩ := hs.ok + refine ⟨hs.t.step hp hv hP tk, L.keepHint hP hh hs.hint, fun i hi => L.keepPoly hP (zk i hi) (hs.z i hi), + by rw [L.keepBytes hP kSB, hs.rho], fun k hk => by rw [L.keepBytes hP (k4 k hk), hs.rho4 k hk], + fun r' hr' c' hc' hd => L.keepRed hP (kE r' hr' c' hc' hd rfl) (hs.red r' hr' c' hc' hd), + q, by rw [h15, hq], fun hq' r' hr' c' hc' hd => ?_, hbad⟩ + obtain ⟨b, hb⟩ := hok hq' r' hr' c' hc' hd + exact ⟨b, by rw [hb, L.keepPolyAt hP (kE r' hr' c' hc' hd rfl)]⟩ + +/-- The two bytes of seed `j` of `SB4`. -/ +abbrev wsS (j : Nat) : List (Ptr × Nat) := [(sc (oSB4 + 34 * j + 32), 1), (sc (oSB4 + 34 * j + 33), 1)] + +/-- What setting the bytes of seed `j` needs, after `(r, c)`. -/ +def slotChk (p : Params) (r c j : Nat) : Bool := + inB (vW p) (sc (oSB4 + 34 * j + 32)) 1 && inB (vW p) (sc (oSB4 + 34 * j + 33)) 1 && + s3Chk p r c (fun _ _ => false) (wsS j) && + (List.range j).all (fun k => keepB (vB p) (wsS j) (sc (oSB4 + 34 * k)) 34) + +/-- After the bytes of the first `j` seeds of `SB4` for the entries `(r, c₀ + k)`. -/ +structure GS (p : Params) (h : List (Vector Bool n)) (r c c₀ j : Nat) (σ st : State) : Prop where + s3 : S3 p h r c σ st + done : ∀ k < j, bytesAt st.mem (pa st (sc (oSB4 + 34 * k))) 34 = aSeed (vPk p σ) r (c₀ + k) + +theorem slot_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} + {r c c₀ j : Nat} (hr : r < 256) (hj : j < 4) (hx : c₀ + j < 256) (hck : slotChk p r c j = true) {s : State} + (hs : GS p h r c c₀ j σ s) : WP isa (.block (setSR r c₀ j)) s (GS p h r c c₀ (j + 1) σ) := by + simp only [slotChk, Bool.and_eq_true, List.all_eq_true, List.mem_range] at hck + obtain ⟨⟨⟨h32, h33⟩, h3⟩, hk⟩ := hck + have L := hs.s3.t.lay hp hv + unfold setSR + refine WP.mono (setB2At_ok L (oSB4 + 34 * j) h32 h33 (c₀ + j) r hx hr) fun s₁ ⟨hP₁, h15₁, hm₁⟩ => + ⟨hs.s3.keep hp hv hP₁ h15₁ h3, fun k hk' => ?_⟩ + rcases (by omega : k < j ∨ k = j) with hk' | rfl + · rw [L.keepBytes hP₁ (hk k hk'), hs.done k hk'] + · rw [hP₁.pa (show Reg.rbx ∈ bases by decide), hm₁, seed_bytes, hs.s3.rho4 k hj, aSeed, integerToBytes_one, integerToBytes_one] + + +/-- Whether `(r', c')` is one of the four entries from `(r, c₀)`. -/ +abbrev inGrp (r c₀ r' c' : Nat) : Bool := r' == r && decide (c₀ ≤ c') && decide (c' < c₀ + 4) + +/-- The facts about the parameters the entries `(r, c₀), …, (r, c₀ + 3)` need, after `(r, c)`. -/ +def gChk (p : Params) (r c₀ c : Nat) : Bool := + (List.range 4).all (fun j => slotChk p r c j) && rej4Chk (vB p) (vW p) (pA r c₀) (sc (oR4 p)) && + inB (vB p) (pA r c₀) 4096 && inB (vW p) (pA r c₀) 4096 && + s3Chk p r c (inGrp r c₀) [(pA r c₀, 4096), (sc (oR4 p), 8192)] && + decide (c₀ + 4 ≤ p.ℓ) && decide (c₀ ≤ c) && decide (c ≤ c₀ + 4) + +theorem gChk_all : ∀ p ∈ params, ∀ r < p.k, gChk p r 0 0 = true ∧ (p.ℓ = 7 → gChk p r 3 4 = true) := by decide + +theorem pa_poly4 (s : State) (r c k : Nat) : poly4 (pa s (pA r c)) k = pa s (pA r (c + k)) := by + unfold poly4 + simp only [pa] + rw [BitVec.add_assoc, ← BitVec.ofNat_add, show oP (20 + 8 * r + c) + 1024 * k = oP (20 + 8 * r + (c + k)) by + simp only [oP]; omega] + +theorem aGrp_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) + {h : List (Vector Bool n)} {r c₀ c : Nat} (hr : r < p.k) (hck : gChk p r c₀ c = true) {s : State} + (hs : S3 p h r c σ s) : WP isa (aGrp P p r c₀) s (S3 p h r (c₀ + 4) σ) := by + have hkl := kl_le p hp + simp only [gChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, decide_eq_true_eq] at hck + obtain ⟨⟨⟨⟨⟨⟨⟨hsl, hrej⟩, hin⟩, hwa⟩, hG⟩, hl⟩, hc₀⟩, hc4⟩ := hck + obtain ⟨hk, kSB, k4, kE⟩ := s3Chk_spec hG + obtain ⟨tk, hh, zk⟩ := keepChk_spec hk + unfold aGrp + refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 0 (by decide)) + ⟨hs, fun _ h => absurd h (by omega)⟩) fun _ g₁ => ?_) + refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 1 (by decide)) g₁) fun _ g₂ => ?_) + refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 2 (by decide)) g₂) fun _ g₃ => ?_) + refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 3 (by decide)) g₃) fun s₄ g₄ => ?_) + have L₄ := g₄.s3.t.lay hp hv + obtain ⟨q, h15, hok, hbad⟩ := g₄.s3.ok + have hseed : ∀ k < 4, seed4 s₄.mem (pa s₄ (sc oSB4)) k = aSeed (vPk p σ) r (c₀ + k) := fun k hk => by + unfold seed4 + rw [show pa s₄ (sc oSB4) + BitVec.ofNat 64 (34 * k) = pa s₄ (sc (oSB4 + 34 * k)) by + simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add]] + exact g₄.done k hk + refine WP.mono (sampled4_ok L₄ hin hwa h15 (List.mem_cons_self ..) + (F := fun k b => rejNTTPoly b.rejNTT (aSeed (vPk p σ) r (c₀ + k))) + (WP.mono (rej4At_ok C.rej4 L₄ hrej) fun s' ⟨hP, h15', hred, ho⟩ => ⟨hP, h15', hred, ?_⟩)) + fun s₅ ⟨hP₅, hred₅, rr, hrr, h15₅, hs1, hs0⟩ => ?_ + · rcases ho with ⟨h1, hb⟩ | ⟨h0, k, hk, hn⟩ + · exact .inl ⟨h1, fun k hk => by rw [← hseed k hk]; exact hb k hk⟩ + · exact .inr ⟨h0, k, hk, by rw [← hseed k hk]; exact hn⟩ + have e₅ : ∀ k, poly4 (pa s₄ (pA r c₀)) k = pa s₅ (pA r (c₀ + k)) := fun k => by + rw [pa_poly4, hP₅.pa (show Reg.rbx ∈ bases by decide)] + have hgrp : ∀ c', inGrp r c₀ r c' = true → c₀ ≤ c' ∧ c' < c₀ + 4 := fun c' hg => by + simp only [inGrp, Bool.and_eq_true, beq_self_eq_true, decide_eq_true_eq, true_and] at hg; exact hg + have hout : ∀ r' c', Done r (c₀ + 4) r' c' → inGrp r c₀ r' c' = false → Done r c r' c' := fun r' c' hd hx => by + unfold Done at hd ⊢ + rcases hd with hd | ⟨rfl, hd⟩ + · exact .inl hd + · refine .inr ⟨rfl, ?_⟩ + simp only [inGrp, beq_self_eq_true, Bool.true_and, Bool.and_eq_false_iff, decide_eq_false_iff_not] at hx + omega + refine ⟨g₄.s3.t.step hp hv hP₅ tk, L₄.keepHint hP₅ hh g₄.s3.hint, + fun i hi => L₄.keepPoly hP₅ (zk i hi) (g₄.s3.z i hi), by rw [L₄.keepBytes hP₅ kSB, g₄.s3.rho], + fun k hk => by rw [L₄.keepBytes hP₅ (k4 k hk), g₄.s3.rho4 k hk], fun r' hr' c' hc' hd => ?_, + q && (rr == 1), ?_, fun hq r' hr' c' hc' hd => ?_, fun hq => ?_⟩ + · cases hx : inGrp r c₀ r' c' + · exact L₄.keepRed hP₅ (kE r' hr' c' hc' (hout r' c' hd hx) hx) (g₄.s3.red r' hr' c' hc' (hout r' c' hd hx)) + · have e : r' = r := by simp only [inGrp, Bool.and_eq_true, beq_iff_eq] at hx; exact hx.1.1 + subst e + obtain ⟨g1, g2⟩ := hgrp c' hx + have := hred₅ (c' - c₀) (by omega) + rwa [e₅, show c₀ + (c' - c₀) = c' by omega] at this + · rw [h15₅] + exact flag_congr (by cases q <;> simp) + · simp only [Bool.and_eq_true, beq_iff_eq] at hq + cases hx : inGrp r c₀ r' c' + · obtain ⟨b, hb⟩ := hok hq.1 r' hr' c' hc' (hout r' c' hd hx) + exact ⟨b, by rw [hb, L₄.keepPolyAt hP₅ (kE r' hr' c' hc' (hout r' c' hd hx) hx)]⟩ + · have e : r' = r := by simp only [inGrp, Bool.and_eq_true, beq_iff_eq] at hx; exact hx.1.1 + subst e + obtain ⟨g1, g2⟩ := hgrp c' hx + obtain ⟨b, hb⟩ := hs1 hq.2 (c' - c₀) (by omega) + refine ⟨b, ?_⟩ + rwa [e₅, show c₀ + (c' - c₀) = c' by omega] at hb + · cases hq' : q + · obtain ⟨r', hr', c', hc', hd, hn⟩ := hbad hq' + refine ⟨r', hr', c', hc', ?_, hn⟩ + unfold Done at hd ⊢; omega + · rw [hq'] at hq + have h0 : rr = 0 := by + rcases hrr with h1 | h0 + · rw [h1] at hq; cases hq + · exact h0 + obtain ⟨k, hk, hn⟩ := hs0 h0 + exact ⟨r, hr, c₀ + k, by omega, by unfold Done; omega, hn⟩ + theorem aRow_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {r : Nat} (hr : r < p.k) {s : State} (hs : S3 p h r 0 σ s) : WP isa (aRow P p r) s (S3 p h (r + 1) 0 σ) := by + have hkl := kl_le p hp + have hg := gChk_all p hp r hr + have hl : p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by + have : ∀ p ∈ params, p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by decide + exact this p hp unfold aRow - refine WP.mono (seqR_ok (I := fun e => S3 p h r (e - 8 * r) σ) p.ℓ (8 * r) (fun e he he' st hst => ?_) s - (by rw [Nat.sub_self]; exact hs)) fun st hst => ?_ - · have := aOne_ok C hp hv hr (c := e - 8 * r) (by omega) hst - rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this - exact this - · rw [show 8 * r + p.ℓ - 8 * r = p.ℓ by omega] at hst - exact hst.next + refine WP.seq (WP.mono (aGrp_ok C hp hv hr hg.1 hs) fun s₁ h₁ => ?_) + rw [Nat.zero_add] at h₁ + by_cases h7 : p.ℓ = 7 + · rw [ite_eq_left h7] + refine WP.mono (aGrp_ok C hp hv hr (hg.2 h7) h₁) fun s₂ h₂ => ?_ + rw [show 3 + 4 = p.ℓ by omega] at h₂ + exact h₂.next + · rw [ite_eq_right h7] + refine WP.mono (seqR_ok (I := fun e => S3 p h r (e - 8 * r) σ) (p.ℓ - 4) (8 * r + 4) + (fun e he he' st hst => ?_) s₁ (by rw [show 8 * r + 4 - 8 * r = 4 by omega]; exact h₁)) fun st hst => ?_ + · have := aOne_ok C hp hv hr (c := e - 8 * r) (by omega) hst + rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this + exact this + · rw [show 8 * r + 4 + (p.ℓ - 4) - 8 * r = p.ℓ by omega] at hst + exact hst.next /-- After the samplers, with the hint `h`. -/ structure S4 (p : Params) (h : List (Vector Bool n)) (σ st : State) : Prop where @@ -223,9 +408,18 @@ def sChk (p : Params) : Bool := theorem sChk_all : ∀ p ∈ params, sChk p = true := by decide +/-- After `ρ` is copied to `SB` and the first `j` seeds of `SB4`. -/ +structure RhoS (p : Params) (h : List (Vector Bool n)) (j : Nat) (σ st : State) : Prop where + t : T p σ st + hint : HintIs st.mem (pa st (pH 0)) p.k h + z : ∀ i < p.ℓ, PolyIs st.mem (pa st (pZ i)) (toRq (vZ p (vSig p σ) i)) + rho : bytesAt st.mem (pa st (sc oSB)) 32 = vRho (vPk p σ) + rho4 : ∀ k < j, bytesAt st.mem (pa st (sc (oSB4 + 34 * k))) 32 = vRho (vPk p σ) + f15 : st.gpr .r15 = flag True + theorem copyRho_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {s : State} (hs : S2 p h p.ℓ σ s) (h15 : s.gpr .r15 = flag True) : - WP isa (copy (sc oSB) (.rbp, 0) 32) s (S3 p h 0 0 σ) := by + WP isa (copy (sc oSB) (.rbp, 0) 32) s (RhoS p h 0 σ) := by have hc := sChk_all p hp simp only [sChk, Bool.and_eq_true, decide_eq_true_eq, List.all_eq_true, List.mem_range] at hc obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨c1, c2⟩, c3⟩, c4⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hc @@ -234,11 +428,53 @@ theorem copyRho_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ refine WP.mono (copy_ok L (by decide) c1 c2) fun s₁ ⟨hP₁, h15₁, hcp⟩ => ?_ have t₁ := hs.t.step hp hv hP₁ t3 refine ⟨t₁, L.keepHint hP₁ h3 hs.hint, fun i hi => L.keepPoly hP₁ (z3 i hi) (hs.z i hi), ?_, - fun r' _ c' _ hd => absurd hd (by unfold Done; omega), true, by rw [h15₁, h15]; exact flag_congr (by simp), - fun _ r' _ c' _ hd => absurd hd (by unfold Done; omega), fun hq => absurd hq (by simp)⟩ + fun _ h => absurd h (Nat.not_lt_zero _), by rw [h15₁, h15]⟩ rw [hP₁.pa (show Reg.rbx ∈ bases by decide), hcp, hs.t.pkSlice c4, List.drop_zero] rfl +/-- What copying `ρ` to seed `j` of `SB4` needs. -/ +def rChk (p : Params) (j : Nat) : Bool := + sepB (vB p) (.rbp, 0) 32 (sc (oSB4 + 34 * j)) 32 && inB (vW p) (sc (oSB4 + 34 * j)) 32 && + keepChk p [(sc (oSB4 + 34 * j), 32)] && keepB (vB p) [(sc (oSB4 + 34 * j), 32)] (sc oSB) 32 && + (List.range j).all (fun k => keepB (vB p) [(sc (oSB4 + 34 * j), 32)] (sc (oSB4 + 34 * k)) 32) + +theorem rChk_all : ∀ p ∈ params, ∀ j < 4, rChk p j = true := by decide + +theorem copyK_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {j : Nat} + (hj : j < 4) {s : State} (hs : RhoS p h j σ s) : + WP isa (copy (sc (oSB4 + 34 * j)) (.rbp, 0) 32) s (RhoS p h (j + 1) σ) := by + have hc := rChk_all p hp j hj + simp only [rChk, Bool.and_eq_true, List.all_eq_true, List.mem_range] at hc + obtain ⟨⟨⟨⟨c1, c2⟩, c3⟩, c4⟩, c5⟩ := hc + have hpk : 32 ≤ p.pkLen := by + have := sChk_all p hp + simp only [sChk, Bool.and_eq_true, decide_eq_true_eq] at this + exact this.1.1.1.1.1.1.1.2 + obtain ⟨t3, h3, z3⟩ := keepChk_spec c3 + have L := hs.t.lay hp hv + refine WP.mono (copy_ok L (by decide) c1 c2) fun s₁ ⟨hP₁, h15₁, hcp⟩ => ?_ + refine ⟨hs.t.step hp hv hP₁ t3, L.keepHint hP₁ h3 hs.hint, fun i hi => L.keepPoly hP₁ (z3 i hi) (hs.z i hi), + by rw [L.keepBytes hP₁ c4, hs.rho], fun k hk => ?_, by rw [h15₁, hs.f15]⟩ + rcases (by omega : k < j ∨ k = j) with hk | rfl + · rw [L.keepBytes hP₁ (c5 k hk), hs.rho4 k hk] + · rw [hP₁.pa (show Reg.rbx ∈ bases by decide), hcp, hs.t.pkSlice hpk, List.drop_zero] + rfl + +theorem RhoS.s3 {p : Params} {h : List (Vector Bool n)} {σ s : State} (r : RhoS p h 4 σ s) : S3 p h 0 0 σ s := + ⟨r.t, r.hint, r.z, r.rho, r.rho4, fun r' _ c' _ hd => absurd hd (by unfold Done; omega), + true, by rw [r.f15]; exact flag_congr (by simp), fun _ r' _ c' _ hd => absurd hd (by unfold Done; omega), + fun hq => absurd hq (by simp)⟩ + +theorem rhos_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) + {h : List (Vector Bool n)} {s : State} (hs : S2 p h p.ℓ σ s) (h15 : s.gpr .r15 = flag True) : + WP isa rhos s (S3 p h 0 0 σ) := by + unfold rhos + refine WP.seq (WP.mono (copyRho_ok hp hv hs h15) fun s₀ r₀ => ?_) + refine WP.seq (WP.mono (copyK_ok hp hv (j := 0) (by decide) r₀) fun s₁ r₁ => ?_) + refine WP.seq (WP.mono (copyK_ok hp hv (j := 1) (by decide) r₁) fun s₂ r₂ => ?_) + refine WP.seq (WP.mono (copyK_ok hp hv (j := 2) (by decide) r₂) fun s₃ r₃ => ?_) + exact WP.mono (copyK_ok hp hv (j := 3) (by decide) r₃) fun _ r₄ => r₄.s3 + theorem ballStage_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {s₂ : State} (hs₂ : S3 p h p.k 0 σ s₂) : WP isa (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC) s₂ (S4 p h σ) := by @@ -279,7 +515,7 @@ theorem samples_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {h : List (Vector Bool n)} {s : State} (hs : S2 p h p.ℓ σ s) (h15 : s.gpr .r15 = flag True) : WP isa (samples P p) s (S4 p h σ) := by unfold samples - refine WP.seq (WP.mono (copyRho_ok hp hv hs h15) fun s₁ s₁3 => ?_) + refine WP.seq (WP.mono (rhos_ok hp hv hs h15) fun s₁ s₁3 => ?_) refine WP.seq (WP.mono (seqR_ok (I := fun r => S3 p h r 0 σ) p.k 0 (fun r _ hr st hst => aRow_ok C hp hv (by omega) hst) s₁ s₁3) fun s₂ hs₂ => ?_) rw [Nat.zero_add] at hs₂ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Top.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Top.lean index e623e7872..6369ce2eb 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Top.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Top.lean @@ -6,7 +6,7 @@ import VerifiedGarbage.Spec.MlDsa.Contract # ML-DSA verification on x86-64: the contract, the layout and the invariant Untrusted: everything here is checked by Lean. The precondition of the -shared contract `verifyContract p X86_64.abi 24`, spelled out (`VPre`); the +shared contract `verifyContract p X86_64.abi 32`, spelled out (`VPre`); the layout of the function's buffers (`pk`, `mu`, `sig` read in `rbp`, `r12`, `r13`; `scratch` written, in `rbx`: `vR p`, `vW p`); what holds throughout (`T`: the permissions and stack pointer of entry, the pointers, the caller's @@ -28,9 +28,9 @@ def params : List Params := [mlDsa44, mlDsa65, mlDsa87] /-- The size of `scratch` in bytes. -/ abbrev scrLen (p : Params) : Nat := scratchWords p * 8 -/-- The precondition of `verifyContract p X86_64.abi 24`. -/ +/-- The precondition of `verifyContract p X86_64.abi 32`. -/ structure VPre (p : Params) (σ : State) : Prop where - sp : 24 ≤ (σ.gpr .rsp).toNat + sp : 32 ≤ (σ.gpr .rsp).toNat rd : σ.rd = [⟨σ.gpr .rdi, p.pkLen⟩, ⟨σ.gpr .rsi, 64⟩, ⟨σ.gpr .rdx, p.sigLen⟩] wr : σ.wr = [⟨σ.gpr .rcx, scrLen p⟩] d1 : Region.Disjoint ⟨σ.gpr .rdi, p.pkLen⟩ ⟨σ.gpr .rcx, scrLen p⟩ @@ -40,10 +40,10 @@ structure VPre (p : Params) (σ : State) : Prop where r2 : Region.Disjoint ⟨σ.gpr .rsp, 8⟩ ⟨σ.gpr .rsi, 64⟩ r3 : Region.Disjoint ⟨σ.gpr .rsp, 8⟩ ⟨σ.gpr .rdx, p.sigLen⟩ r4 : Region.Disjoint ⟨σ.gpr .rsp, 8⟩ ⟨σ.gpr .rcx, scrLen p⟩ - k1 : (below (σ.gpr .rsp) 24).Disjoint ⟨σ.gpr .rdi, p.pkLen⟩ - k2 : (below (σ.gpr .rsp) 24).Disjoint ⟨σ.gpr .rsi, 64⟩ - k3 : (below (σ.gpr .rsp) 24).Disjoint ⟨σ.gpr .rdx, p.sigLen⟩ - k4 : (below (σ.gpr .rsp) 24).Disjoint ⟨σ.gpr .rcx, scrLen p⟩ + k1 : (below (σ.gpr .rsp) 32).Disjoint ⟨σ.gpr .rdi, p.pkLen⟩ + k2 : (below (σ.gpr .rsp) 32).Disjoint ⟨σ.gpr .rsi, 64⟩ + k3 : (below (σ.gpr .rsp) 32).Disjoint ⟨σ.gpr .rdx, p.sigLen⟩ + k4 : (below (σ.gpr .rsp) 32).Disjoint ⟨σ.gpr .rcx, scrLen p⟩ n1 : (σ.gpr .rdi).toNat + p.pkLen ≤ 2 ^ 64 n2 : (σ.gpr .rsi).toNat + 64 ≤ 2 ^ 64 n3 : (σ.gpr .rdx).toNat + p.sigLen ≤ 2 ^ 64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Verified.lean index 37d95f25b..f87b8ef96 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Verified.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Verified.lean @@ -70,7 +70,7 @@ def verifySat (p : Params) : State where rd := [⟨0x10000, p.pkLen⟩, ⟨0x20000, 64⟩, ⟨0x30000, p.sigLen⟩] wr := [⟨0x40000, scrLen p⟩] -theorem verify_sat : ∀ p ∈ params, ∃ s, (verifyContract p X86_64.abi 24).pre s := by +theorem verify_sat : ∀ p ∈ params, ∃ s, (verifyContract p X86_64.abi 32).pre s := by intro p hp simp only [params, List.mem_cons, List.not_mem_nil, or_false] at hp rcases hp with rfl | rfl | rfl @@ -78,7 +78,7 @@ theorem verify_sat : ∀ p ∈ params, ∃ s, (verifyContract p X86_64.abi 24).p · sig_implies_sat [verifyContract, verifySig, X86_64.abi, VG.X86_64.argRegs] [verifySat] using verifySat mlDsa65 · sig_implies_sat [verifyContract, verifySig, X86_64.abi, VG.X86_64.argRegs] [verifySat] using verifySat mlDsa87 -theorem verify_implies {p : Params} (hp : p ∈ params) : (verifyK p).Implies (verifyContract p X86_64.abi 24) where +theorem verify_implies {p : Params} (hp : p ∈ params) : (verifyK p).Implies (verifyContract p X86_64.abi 32) where pre s h := by sig_pre [verifyContract, verifySig, X86_64.abi, VG.X86_64.argRegs] at h obtain ⟨a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18⟩ := h @@ -95,9 +95,9 @@ theorem verify_implies {p : Params} (hp : p ∈ params) : (verifyK p).Implies (v exact ⟨hdi, hsi, hdx, hcx, hsp, e₁, e₂, e₃⟩ sat := verify_sat p hp -/-- `verify P p` meets `verifyContract p` with 24 bytes of stack. -/ +/-- `verify P p` meets `verifyContract p` with 32 bytes of stack. -/ theorem verify_verified {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) : - Verified X86_64.target (verify P p) (verifyContract p X86_64.abi 24) := + Verified X86_64.target (verify P p) (verifyContract p X86_64.abi 32) := Verified.of_correct (verify_correct C hp) (verify_ct C hp) (verify_implies hp) end VG.Proof.MlDsa.X86_64.Verify diff --git a/src/asm/x86_64/mldsa.rs b/src/asm/x86_64/mldsa.rs index 541b9d25a..2997f10a4 100644 --- a/src/asm/x86_64/mldsa.rs +++ b/src/asm/x86_64/mldsa.rs @@ -4933,6 +4933,4847 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_rej_ntt_poly(seed: *const [u8; 34] ) } +/// `RejNTTPoly` (FIPS 204 Algorithm 30) four times: for each `k` < 4, writes the element of `T_q` sampled from the SHAKE128 output of the 34 bytes of `*seeds` from byte `34 k` to the 256 coefficients of `*a` from coefficient `256 k` (each less than `q` = 8380417), and returns 1. Returns 0 if the loop reaches its bound, which is at least 894 bytes of SHAKE128 output for each (FIPS 204 Appendix C; this happens with probability about 2^-256 or less): `*a` is then unspecified, and the caller must destroy it and treat the operation as failed. The four are independent, so an implementation may compute them together (e.g. four SHAKE128 instances at once in vector registers). +/// +/// Contract: `VG.Spec.MlDsa.rejNTT4Contract`. Not constant time in the seeds: timing may depend on the pointers and on `*seeds` (public in ML-DSA: the seed `ρ` of the matrix and indices), but not on anything else. +/// +/// It calls `vg_mldsa_rej_ntt_poly` on each seed, and saves its caller's callee-saved registers in `scratch`. +/// +/// # Safety +/// +/// * `seeds` must be valid for reads of 136 bytes. +/// * `a` must be valid for reads and writes of 4096 bytes. +/// * `scratch` must be valid for reads and writes of 8192 bytes. +/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `a` and `scratch` must not overlap each other or `seeds` (distinct Rust objects never do). +/// * None of `seeds`, `a` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_rej_ntt_poly4(seeds: *const [u8; 136], a: *mut [u32; 1024], scratch: *mut [u64; 1024]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rdx+4384], rbx", + "mov QWORD PTR [rdx+4392], rbp", + "mov QWORD PTR [rdx+4400], r12", + "mov QWORD PTR [rdx+4408], r13", + "mov QWORD PTR [rdx+4416], r14", + "mov rbx, rdx", + "mov r12, rdi", + "mov r13, rsi", + "mov r14d, 1", + "mov rdi, r12", + "add rdi, 0", + "mov rsi, r13", + "add rsi, 0", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_rej_ntt_poly}", + "and r14d, eax", + "mov rdi, r12", + "add rdi, 34", + "mov rsi, r13", + "add rsi, 1024", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_rej_ntt_poly}", + "and r14d, eax", + "mov rdi, r12", + "add rdi, 68", + "mov rsi, r13", + "add rsi, 2048", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_rej_ntt_poly}", + "and r14d, eax", + "mov rdi, r12", + "add rdi, 102", + "mov rsi, r13", + "add rsi, 3072", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_rej_ntt_poly}", + "and r14d, eax", + "mov eax, r14d", + "mov r14, QWORD PTR [rbx+4416]", + "mov r13, QWORD PTR [rbx+4408]", + "mov r12, QWORD PTR [rbx+4400]", + "mov rbp, QWORD PTR [rbx+4392]", + "mov rbx, QWORD PTR [rbx+4384]", + "ret", + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + ) +} + +/// The CPU features `vg_mldsa_rej_ntt_poly4_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_REJ_NTT_POLY4_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// `RejNTTPoly` (FIPS 204 Algorithm 30) four times: for each `k` < 4, writes the element of `T_q` sampled from the SHAKE128 output of the 34 bytes of `*seeds` from byte `34 k` to the 256 coefficients of `*a` from coefficient `256 k` (each less than `q` = 8380417), and returns 1. Returns 0 if the loop reaches its bound, which is at least 894 bytes of SHAKE128 output for each (FIPS 204 Appendix C; this happens with probability about 2^-256 or less): `*a` is then unspecified, and the caller must destroy it and treat the operation as failed. The four are independent, so an implementation may compute them together (e.g. four SHAKE128 instances at once in vector registers). +/// +/// Contract: `VG.Spec.MlDsa.rejNTT4Contract`. Not constant time in the seeds: timing may depend on the pointers and on `*seeds` (public in ML-DSA: the seed `ρ` of the matrix and indices), but not on anything else. +/// +/// It runs the four instances of SHAKE128 at once, in the four 64-bit elements of AVX2 registers (as `vg_mlkem_sample_ntt4_avx2` does), squeezing three blocks of each twice, and runs the loop of `RejNTTPoly` over the 1008 bytes of each seed's output, as `vg_mldsa_rej_ntt_poly` does. It saves its caller's callee-saved registers in `scratch`. +/// +/// # Safety +/// +/// * `seeds` must be valid for reads of 136 bytes. +/// * `a` must be valid for reads and writes of 4096 bytes. +/// * `scratch` must be valid for reads and writes of 8192 bytes. +/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `a` and `scratch` must not overlap each other or `seeds` (distinct Rust objects never do). +/// * None of `seeds`, `a` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_rej_ntt_poly4_avx2(seeds: *const [u8; 136], a: *mut [u32; 1024], scratch: *mut [u64; 1024]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rdx+4384], rbx", + "mov QWORD PTR [rdx+4392], rbp", + "mov QWORD PTR [rdx+4400], r12", + "mov QWORD PTR [rdx+4408], r13", + "mov QWORD PTR [rdx+4416], r14", + "mov rbx, rdx", + "mov r12, rdi", + "mov r13, rsi", + "mov r14d, 1", + "movabs rax, 1", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1600], ymm0", + "movabs rax, 32898", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1632], ymm0", + "movabs rax, -9223372036854742902", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1664], ymm0", + "movabs rax, -9223372034707259392", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1696], ymm0", + "movabs rax, 32907", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1728], ymm0", + "movabs rax, 2147483649", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1760], ymm0", + "movabs rax, -9223372034707259263", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1792], ymm0", + "movabs rax, -9223372036854743031", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1824], ymm0", + "movabs rax, 138", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1856], ymm0", + "movabs rax, 136", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1888], ymm0", + "movabs rax, 2147516425", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1920], ymm0", + "movabs rax, 2147483658", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1952], ymm0", + "movabs rax, 2147516555", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1984], ymm0", + "movabs rax, -9223372036854775669", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2016], ymm0", + "movabs rax, -9223372036854742903", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2048], ymm0", + "movabs rax, -9223372036854743037", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2080], ymm0", + "movabs rax, -9223372036854743038", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2112], ymm0", + "movabs rax, -9223372036854775680", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2144], ymm0", + "movabs rax, 32778", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2176], ymm0", + "movabs rax, -9223372034707292150", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2208], ymm0", + "movabs rax, -9223372034707259263", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2240], ymm0", + "movabs rax, -9223372036854742912", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2272], ymm0", + "movabs rax, 2147483649", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2304], ymm0", + "movabs rax, -9223372034707259384", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2336], ymm0", + "vpxor ymm0, ymm0, ymm0", + "vmovdqu YMMWORD PTR [rbx], ymm0", + "vmovdqu YMMWORD PTR [rbx+32], ymm0", + "vmovdqu YMMWORD PTR [rbx+64], ymm0", + "vmovdqu YMMWORD PTR [rbx+96], ymm0", + "vmovdqu YMMWORD PTR [rbx+128], ymm0", + "vmovdqu YMMWORD PTR [rbx+160], ymm0", + "vmovdqu YMMWORD PTR [rbx+192], ymm0", + "vmovdqu YMMWORD PTR [rbx+224], ymm0", + "vmovdqu YMMWORD PTR [rbx+256], ymm0", + "vmovdqu YMMWORD PTR [rbx+288], ymm0", + "vmovdqu YMMWORD PTR [rbx+320], ymm0", + "vmovdqu YMMWORD PTR [rbx+352], ymm0", + "vmovdqu YMMWORD PTR [rbx+384], ymm0", + "vmovdqu YMMWORD PTR [rbx+416], ymm0", + "vmovdqu YMMWORD PTR [rbx+448], ymm0", + "vmovdqu YMMWORD PTR [rbx+480], ymm0", + "vmovdqu YMMWORD PTR [rbx+512], ymm0", + "vmovdqu YMMWORD PTR [rbx+544], ymm0", + "vmovdqu YMMWORD PTR [rbx+576], ymm0", + "vmovdqu YMMWORD PTR [rbx+608], ymm0", + "vmovdqu YMMWORD PTR [rbx+640], ymm0", + "vmovdqu YMMWORD PTR [rbx+672], ymm0", + "vmovdqu YMMWORD PTR [rbx+704], ymm0", + "vmovdqu YMMWORD PTR [rbx+736], ymm0", + "vmovdqu YMMWORD PTR [rbx+768], ymm0", + "mov rax, QWORD PTR [r12]", + "mov QWORD PTR [rbx], rax", + "mov rax, QWORD PTR [r12+8]", + "mov QWORD PTR [rbx+32], rax", + "mov rax, QWORD PTR [r12+16]", + "mov QWORD PTR [rbx+64], rax", + "mov rax, QWORD PTR [r12+24]", + "mov QWORD PTR [rbx+96], rax", + "movzx eax, BYTE PTR [r12+32]", + "mov BYTE PTR [rbx+128], al", + "movzx eax, BYTE PTR [r12+33]", + "mov BYTE PTR [rbx+129], al", + "mov rax, QWORD PTR [r12+34]", + "mov QWORD PTR [rbx+8], rax", + "mov rax, QWORD PTR [r12+42]", + "mov QWORD PTR [rbx+40], rax", + "mov rax, QWORD PTR [r12+50]", + "mov QWORD PTR [rbx+72], rax", + "mov rax, QWORD PTR [r12+58]", + "mov QWORD PTR [rbx+104], rax", + "movzx eax, BYTE PTR [r12+66]", + "mov BYTE PTR [rbx+136], al", + "movzx eax, BYTE PTR [r12+67]", + "mov BYTE PTR [rbx+137], al", + "mov rax, QWORD PTR [r12+68]", + "mov QWORD PTR [rbx+16], rax", + "mov rax, QWORD PTR [r12+76]", + "mov QWORD PTR [rbx+48], rax", + "mov rax, QWORD PTR [r12+84]", + "mov QWORD PTR [rbx+80], rax", + "mov rax, QWORD PTR [r12+92]", + "mov QWORD PTR [rbx+112], rax", + "movzx eax, BYTE PTR [r12+100]", + "mov BYTE PTR [rbx+144], al", + "movzx eax, BYTE PTR [r12+101]", + "mov BYTE PTR [rbx+145], al", + "mov rax, QWORD PTR [r12+102]", + "mov QWORD PTR [rbx+24], rax", + "mov rax, QWORD PTR [r12+110]", + "mov QWORD PTR [rbx+56], rax", + "mov rax, QWORD PTR [r12+118]", + "mov QWORD PTR [rbx+88], rax", + "mov rax, QWORD PTR [r12+126]", + "mov QWORD PTR [rbx+120], rax", + "movzx eax, BYTE PTR [r12+134]", + "mov BYTE PTR [rbx+152], al", + "movzx eax, BYTE PTR [r12+135]", + "mov BYTE PTR [rbx+153], al", + "mov eax, 31", + "mov BYTE PTR [rbx+130], al", + "mov BYTE PTR [rbx+138], al", + "mov BYTE PTR [rbx+146], al", + "mov BYTE PTR [rbx+154], al", + "mov eax, 128", + "mov BYTE PTR [rbx+647], al", + "mov BYTE PTR [rbx+655], al", + "mov BYTE PTR [rbx+663], al", + "mov BYTE PTR [rbx+671], al", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "20:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 20b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2368], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2376], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2384], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2392], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2400], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2408], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2416], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2424], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2432], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2440], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2448], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2456], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2464], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2472], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2480], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2488], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2496], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2504], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2512], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2520], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2528], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+2872], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+2880], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+2888], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+2896], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+2904], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+2912], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+2920], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+2928], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+2936], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+2944], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+2952], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+2960], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+2968], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+2976], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+2984], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+2992], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3000], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3008], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3016], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3024], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3032], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3376], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3384], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3392], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3400], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3408], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3416], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3424], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3432], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3440], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3448], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3456], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3464], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3472], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3480], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3488], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3496], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3504], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3512], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3520], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3528], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3536], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+3880], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+3888], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+3896], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+3904], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+3912], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+3920], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+3928], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+3936], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+3944], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+3952], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+3960], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+3968], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+3976], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+3984], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+3992], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4000], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4008], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4016], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4024], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4032], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4040], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "21:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 21b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2536], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2544], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2552], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2560], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2568], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2576], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2584], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2592], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2600], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2608], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2616], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2624], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2632], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2640], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2648], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2656], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2664], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2672], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2680], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2688], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2696], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3040], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3048], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3056], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3064], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3072], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3080], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3088], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3096], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3104], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3112], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3120], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3128], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3136], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3144], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3152], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3160], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3168], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3176], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3184], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3192], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3200], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3544], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3552], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3560], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3568], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3576], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3584], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3592], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3600], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3608], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3616], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3624], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3632], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3640], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3648], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3656], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3664], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3672], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3680], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3688], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3696], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3704], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4048], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4056], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4064], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4072], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4080], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4088], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4096], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4104], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4112], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4120], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4128], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4136], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4144], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4152], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4160], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4168], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4176], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4184], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4192], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4200], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4208], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "22:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 22b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2704], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2712], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2720], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2728], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2736], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2744], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2752], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2760], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2768], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2776], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2784], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2792], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2800], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2808], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2816], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2824], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2832], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2840], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2848], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2856], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2864], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3208], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3216], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3224], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3232], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3240], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3248], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3256], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3264], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3272], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3280], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3288], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3296], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3304], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3312], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3320], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3328], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3336], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3344], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3352], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3360], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3368], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3712], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3720], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3728], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3736], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3744], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3752], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3760], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3768], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3776], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3784], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3792], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3800], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3808], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3816], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3824], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3832], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3840], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3848], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3856], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3864], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3872], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4216], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4224], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4232], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4240], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4248], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4256], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4264], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4272], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4280], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4288], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4296], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4304], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4312], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4320], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4328], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4336], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4344], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4352], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4360], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4368], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4376], rax", + "mov eax, 0", + "mov QWORD PTR [rbx+4424], rax", + "mov QWORD PTR [rbx+4432], rax", + "mov QWORD PTR [rbx+4440], rax", + "mov QWORD PTR [rbx+4448], rax", + "mov rsi, rbx", + "add rsi, 2368", + "mov rbp, r13", + "add rbp, 0", + "mov rdi, QWORD PTR [rbx+4424]", + "mov ecx, 168", + "23:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 24f", + "jmp 25f", + "24:", + "cmp r8d, 8380417", + "jb 26f", + "jmp 27f", + "26:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "27:", + "25:", + "add rsi, 3", + "sub rcx, 1", + "jne 23b", + "mov QWORD PTR [rbx+4424], rdi", + "mov rsi, rbx", + "add rsi, 2872", + "mov rbp, r13", + "add rbp, 1024", + "mov rdi, QWORD PTR [rbx+4432]", + "mov ecx, 168", + "28:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 29f", + "jmp 210f", + "29:", + "cmp r8d, 8380417", + "jb 211f", + "jmp 212f", + "211:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "212:", + "210:", + "add rsi, 3", + "sub rcx, 1", + "jne 28b", + "mov QWORD PTR [rbx+4432], rdi", + "mov rsi, rbx", + "add rsi, 3376", + "mov rbp, r13", + "add rbp, 2048", + "mov rdi, QWORD PTR [rbx+4440]", + "mov ecx, 168", + "213:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 214f", + "jmp 215f", + "214:", + "cmp r8d, 8380417", + "jb 216f", + "jmp 217f", + "216:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "217:", + "215:", + "add rsi, 3", + "sub rcx, 1", + "jne 213b", + "mov QWORD PTR [rbx+4440], rdi", + "mov rsi, rbx", + "add rsi, 3880", + "mov rbp, r13", + "add rbp, 3072", + "mov rdi, QWORD PTR [rbx+4448]", + "mov ecx, 168", + "218:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 219f", + "jmp 220f", + "219:", + "cmp r8d, 8380417", + "jb 221f", + "jmp 222f", + "221:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "222:", + "220:", + "add rsi, 3", + "sub rcx, 1", + "jne 218b", + "mov QWORD PTR [rbx+4448], rdi", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "223:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 223b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2368], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2376], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2384], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2392], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2400], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2408], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2416], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2424], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2432], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2440], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2448], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2456], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2464], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2472], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2480], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2488], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2496], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2504], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2512], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2520], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2528], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+2872], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+2880], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+2888], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+2896], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+2904], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+2912], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+2920], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+2928], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+2936], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+2944], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+2952], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+2960], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+2968], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+2976], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+2984], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+2992], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3000], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3008], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3016], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3024], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3032], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3376], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3384], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3392], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3400], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3408], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3416], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3424], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3432], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3440], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3448], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3456], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3464], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3472], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3480], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3488], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3496], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3504], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3512], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3520], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3528], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3536], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+3880], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+3888], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+3896], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+3904], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+3912], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+3920], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+3928], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+3936], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+3944], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+3952], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+3960], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+3968], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+3976], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+3984], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+3992], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4000], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4008], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4016], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4024], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4032], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4040], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "224:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 224b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2536], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2544], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2552], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2560], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2568], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2576], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2584], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2592], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2600], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2608], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2616], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2624], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2632], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2640], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2648], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2656], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2664], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2672], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2680], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2688], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2696], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3040], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3048], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3056], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3064], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3072], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3080], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3088], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3096], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3104], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3112], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3120], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3128], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3136], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3144], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3152], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3160], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3168], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3176], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3184], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3192], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3200], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3544], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3552], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3560], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3568], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3576], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3584], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3592], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3600], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3608], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3616], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3624], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3632], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3640], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3648], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3656], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3664], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3672], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3680], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3688], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3696], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3704], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4048], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4056], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4064], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4072], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4080], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4088], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4096], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4104], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4112], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4120], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4128], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4136], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4144], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4152], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4160], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4168], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4176], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4184], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4192], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4200], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4208], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "225:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 225b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2704], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2712], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2720], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2728], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2736], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2744], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2752], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2760], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2768], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2776], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2784], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2792], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2800], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2808], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2816], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2824], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2832], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2840], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2848], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2856], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2864], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3208], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3216], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3224], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3232], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3240], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3248], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3256], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3264], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3272], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3280], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3288], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3296], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3304], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3312], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3320], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3328], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3336], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3344], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3352], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3360], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3368], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3712], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3720], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3728], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3736], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3744], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3752], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3760], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3768], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3776], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3784], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3792], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3800], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3808], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3816], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3824], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3832], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3840], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3848], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3856], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3864], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3872], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4216], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4224], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4232], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4240], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4248], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4256], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4264], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4272], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4280], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4288], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4296], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4304], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4312], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4320], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4328], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4336], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4344], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4352], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4360], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4368], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4376], rax", + "vzeroupper", + "mov rsi, rbx", + "add rsi, 2368", + "mov rbp, r13", + "add rbp, 0", + "mov rdi, QWORD PTR [rbx+4424]", + "mov ecx, 168", + "226:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 227f", + "jmp 228f", + "227:", + "cmp r8d, 8380417", + "jb 229f", + "jmp 230f", + "229:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "230:", + "228:", + "add rsi, 3", + "sub rcx, 1", + "jne 226b", + "mov rax, rdi", + "shr rax, 8", + "and r14d, eax", + "mov rsi, rbx", + "add rsi, 2872", + "mov rbp, r13", + "add rbp, 1024", + "mov rdi, QWORD PTR [rbx+4432]", + "mov ecx, 168", + "231:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 232f", + "jmp 233f", + "232:", + "cmp r8d, 8380417", + "jb 234f", + "jmp 235f", + "234:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "235:", + "233:", + "add rsi, 3", + "sub rcx, 1", + "jne 231b", + "mov rax, rdi", + "shr rax, 8", + "and r14d, eax", + "mov rsi, rbx", + "add rsi, 3376", + "mov rbp, r13", + "add rbp, 2048", + "mov rdi, QWORD PTR [rbx+4440]", + "mov ecx, 168", + "236:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 237f", + "jmp 238f", + "237:", + "cmp r8d, 8380417", + "jb 239f", + "jmp 240f", + "239:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "240:", + "238:", + "add rsi, 3", + "sub rcx, 1", + "jne 236b", + "mov rax, rdi", + "shr rax, 8", + "and r14d, eax", + "mov rsi, rbx", + "add rsi, 3880", + "mov rbp, r13", + "add rbp, 3072", + "mov rdi, QWORD PTR [rbx+4448]", + "mov ecx, 168", + "241:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 242f", + "jmp 243f", + "242:", + "cmp r8d, 8380417", + "jb 244f", + "jmp 245f", + "244:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "245:", + "243:", + "add rsi, 3", + "sub rcx, 1", + "jne 241b", + "mov rax, rdi", + "shr rax, 8", + "and r14d, eax", + "mov eax, r14d", + "mov r14, QWORD PTR [rbx+4416]", + "mov r13, QWORD PTR [rbx+4408]", + "mov r12, QWORD PTR [rbx+4400]", + "mov rbp, QWORD PTR [rbx+4392]", + "mov rbx, QWORD PTR [rbx+4384]", + "ret", + ) +} + /// `RejBoundedPoly` (FIPS 204 Algorithm 31): writes the polynomial with coefficients in `[-eta, eta]` sampled from the SHAKE256 output of the 66 bytes `*seed` to `*a` (each coefficient modulo `q` = 8380417), and returns 1. Returns 0 if the loop reaches its bound, which is at least 481 bytes of SHAKE256 output (FIPS 204 Appendix C; this happens with probability about 2^-256 or less): `*a` is then unspecified, and the caller must destroy it and treat the operation as failed. /// /// Contract: `VG.Spec.MlDsa.rejBoundedContract`. Not constant time in which half-bytes of the SHAKE256 output it rejects: timing may depend on the pointers, `eta`, and whether each half-byte of the first 1088 bytes of output is rejected (`rejBoundedLeak`), which is independent of the coefficients sampled from those accepted; but not on anything else of the seed or the output. diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs index 90e8d05f4..3ac477362 100644 --- a/src/asm/x86_64/mldsa44.rs +++ b/src/asm/x86_64/mldsa44.rs @@ -2537,7 +2537,7 @@ pub(crate) const VG_MLDSA44_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; /// * `scratch` must be valid for reads and writes of 77824 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). /// * The CPU must support the `avx` and `avx2` target features. #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312], mu: *const [u8; 64], sig: *const [u8; 2420], scratch: *mut [u64; 9728]) -> u32 { @@ -2634,119 +2634,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -2754,270 +2718,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rdi, 4", "sub rcx, 1", "jne 29b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "212:", + "add rdi, 36864", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 212b", + "jne 210b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "216:", + "add rdi, 45056", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", + "jne 211b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "220:", + "add rdi, 53248", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 212b", "mov rdi, r13", "add rdi, 0", "mov esi, 32", @@ -3033,13 +2841,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 213b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -3415,7 +3223,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rdi, 0", "mov ecx, 32", "mov edx, 0", - "222:", + "214:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -3423,7 +3231,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 222b", + "jne 214b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -3440,7 +3248,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, @@ -5980,7 +5788,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: /// * `scratch` must be valid for reads and writes of 77824 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu: *const [u8; 64], sig: *const [u8; 2420], scratch: *mut [u64; 9728]) -> u32 { core::arch::naked_asm!( @@ -6076,119 +5884,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6196,270 +5968,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rdi, 4", "sub rcx, 1", "jne 29b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "212:", + "add rdi, 36864", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 212b", + "jne 210b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "216:", + "add rdi, 45056", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", + "jne 211b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "220:", + "add rdi, 53248", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 212b", "mov rdi, r13", "add rdi, 0", "mov esi, 32", @@ -6475,13 +6091,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 213b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -6857,7 +6473,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rdi, 0", "mov ecx, 32", "mov edx, 0", - "222:", + "214:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -6865,7 +6481,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 222b", + "jne 214b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -6882,7 +6498,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs index 6452825aa..f85fb71ee 100644 --- a/src/asm/x86_64/mldsa65.rs +++ b/src/asm/x86_64/mldsa65.rs @@ -3716,7 +3716,7 @@ pub(crate) const VG_MLDSA65_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; /// * `scratch` must be valid for reads and writes of 103424 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). /// * The CPU must support the `avx` and `avx2` target features. #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952], mu: *const [u8; 64], sig: *const [u8; 3309], scratch: *mut [u64; 12928]) -> u32 { @@ -3826,119 +3826,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -3946,14 +3910,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 29b", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 1", + "mov eax, 0", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -3961,7 +3925,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", + "add rdi, 32768", "mov ecx, 256", "210:", "mov eax, DWORD PTR [rdi]", @@ -3970,23 +3934,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 210b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", + "add rdi, 36864", + "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -3994,14 +3970,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 211b", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 1", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4009,7 +3985,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 38912", + "add rdi, 40960", "mov ecx, 256", "212:", "mov eax, DWORD PTR [rdi]", @@ -4018,23 +3994,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 212b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", + "add rdi, 45056", + "mov ecx, 1024", "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4044,12 +4032,12 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "jne 213b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4057,7 +4045,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 40960", + "add rdi, 49152", "mov ecx, 256", "214:", "mov eax, DWORD PTR [rdi]", @@ -4067,22 +4055,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "sub rcx, 1", "jne 214b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", + "add rdi, 53248", + "mov ecx, 1024", "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4090,14 +4090,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 215b", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4105,7 +4105,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 46080", + "add rdi, 57344", "mov ecx, 256", "216:", "mov eax, DWORD PTR [rdi]", @@ -4114,23 +4114,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", + "add rdi, 61440", + "mov ecx, 1024", "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4138,14 +4150,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 217b", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 65536", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4153,7 +4165,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", + "add rdi, 65536", "mov ecx, 256", "218:", "mov eax, DWORD PTR [rdi]", @@ -4162,23 +4174,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 218b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", + "add rdi, 69632", + "mov ecx, 1024", "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4186,14 +4210,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 219b", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4201,7 +4225,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", + "add rdi, 73728", "mov ecx, 256", "220:", "mov eax, DWORD PTR [rdi]", @@ -4210,22 +4234,20 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 220b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "mov rdi, r13", + "add rdi, 0", + "mov esi, 48", + "mov edx, 49", + "mov rcx, rbx", + "add rcx, 23552", + "mov r8, rbx", + "add r8, 4096", + "call {vg_mldsa_sample_in_ball}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 54272", + "add rdi, 23552", "mov ecx, 256", "221:", "mov eax, DWORD PTR [rdi]", @@ -4234,422 +4256,88 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 221b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 16384", "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "222:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 222b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 18432", "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "223:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 223b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 20480", "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "224:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 224b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "225:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 225b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 29696", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 30720", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 31744", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 32768", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 71680", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 72704", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 73728", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov rdi, r13", - "add rdi, 0", - "mov esi, 48", - "mov edx, 49", - "mov rcx, rbx", - "add rcx, 23552", - "mov r8, rbx", - "add r8, 4096", - "call {vg_mldsa_sample_in_ball}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 23552", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov rdi, rbx", - "add rdi, 16384", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 17408", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 18432", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 19456", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 20480", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 23552", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 32", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", - "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 25600", - "mov rsi, rbx", - "add rsi, 23552", - "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", @@ -5130,7 +4818,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 0", "mov ecx, 48", "mov edx, 0", - "236:", + "222:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -5138,7 +4826,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 236b", + "jne 222b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -5155,6 +4843,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, @@ -8874,7 +8563,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: /// * `scratch` must be valid for reads and writes of 103424 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu: *const [u8; 64], sig: *const [u8; 3309], scratch: *mut [u64; 12928]) -> u32 { core::arch::naked_asm!( @@ -8983,398 +8672,98 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", - "29:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 29b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 36864", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 39936", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 40960", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "mov rdi, rbx", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "28:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 219b", + "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "220:", + "add rdi, 28672", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", - "mov eax, 1", + "jne 29b", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 3", + "mov eax, 0", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9382,47 +8771,59 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 54272", + "add rdi, 32768", "mov ecx, 256", - "221:", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 210b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "222:", + "add rdi, 36864", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", - "mov eax, 3", + "jne 211b", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 3", + "mov eax, 1", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9430,47 +8831,59 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", + "add rdi, 40960", "mov ecx, 256", - "223:", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "224:", + "add rdi, 45056", + "mov ecx, 1024", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "jne 213b", "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9478,47 +8891,59 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 61440", + "add rdi, 49152", "mov ecx, 256", - "225:", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", + "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "226:", + "add rdi, 53248", + "mov ecx, 1024", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 226b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "jne 215b", "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9526,39 +8951,51 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 63488", + "add rdi, 57344", "mov ecx, 256", - "227:", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 227b", + "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "228:", + "add rdi, 61440", + "mov ecx, 1024", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 228b", + "jne 217b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 4", @@ -9576,109 +9013,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 65536", "mov ecx, 256", - "229:", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 229b", + "jne 218b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 69632", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 71680", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "233:", + "add rdi, 69632", + "mov ecx, 1024", + "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 233b", + "jne 219b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 5", @@ -9696,13 +9073,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 73728", "mov ecx, 256", - "234:", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 234b", + "jne 220b", "mov rdi, r13", "add rdi, 0", "mov esi, 48", @@ -9718,13 +9095,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "235:", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 235b", + "jne 221b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -10287,7 +9664,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rdi, 0", "mov ecx, 48", "mov edx, 0", - "236:", + "222:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -10295,7 +9672,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 236b", + "jne 222b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -10312,6 +9689,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs index 79f684dd0..50336abce 100644 --- a/src/asm/x86_64/mldsa87.rs +++ b/src/asm/x86_64/mldsa87.rs @@ -5604,7 +5604,7 @@ pub(crate) const VG_MLDSA87_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; /// * `scratch` must be valid for reads and writes of 144384 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). /// * The CPU must support the `avx` and `avx2` target features. #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592], mu: *const [u8; 64], sig: *const [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { @@ -5740,119 +5740,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5860,23 +5824,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 29b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 31744", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 33792", - "mov ecx, 256", + "add rdi, 31744", + "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5884,23 +5860,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 210b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 34816", - "mov ecx, 256", + "add rdi, 36864", + "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5908,23 +5896,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 211b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 39936", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", + "add rdi, 39936", + "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5932,23 +5932,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", + "add rdi, 45056", + "mov ecx, 1024", "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5956,23 +5968,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 213b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 48128", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", + "add rdi, 48128", + "mov ecx, 1024", "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5980,23 +6004,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", + "add rdi, 53248", + "mov ecx, 1024", "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6004,23 +6040,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 215b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 56320", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", + "add rdi, 56320", + "mov ecx, 1024", "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6028,23 +6076,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 216b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 41984", - "mov ecx, 256", + "add rdi, 61440", + "mov ecx, 1024", "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6052,23 +6112,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 217b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 64512", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 43008", - "mov ecx, 256", + "add rdi, 64512", + "mov ecx, 1024", "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6077,22 +6149,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "sub rcx, 1", "jne 218b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", + "mov rdi, rbx", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", + "add rdi, 69632", + "mov ecx, 1024", "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6100,23 +6184,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 219b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 72704", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", + "add rdi, 72704", + "mov ecx, 1024", "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6124,23 +6220,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 220b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", + "add rdi, 77824", + "mov ecx, 1024", "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6149,22 +6257,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "sub rcx, 1", "jne 221b", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 80896", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", + "add rdi, 80896", + "mov ecx, 1024", "222:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6172,23 +6292,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 222b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 86016", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", + "add rdi, 86016", + "mov ecx, 1024", "223:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6196,23 +6328,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 223b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 89088", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 50176", - "mov ecx, 256", + "add rdi, 89088", + "mov ecx, 1024", "224:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6220,22 +6364,20 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 224b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "mov rdi, r13", + "add rdi, 0", + "mov esi, 64", + "mov edx, 60", + "mov rcx, rbx", + "add rcx, 23552", + "mov r8, rbx", + "add r8, 4096", + "call {vg_mldsa_sample_in_ball}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 51200", + "add rdi, 23552", "mov ecx, 256", "225:", "mov eax, DWORD PTR [rdi]", @@ -6244,971 +6386,473 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 225b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 16384", "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 18432", "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 20480", "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 22528", "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 30720", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 32768", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 34816", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1024", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "236:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 236b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 38912", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "237:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 237b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 66560", + "add rsi, 40960", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 66560", - "mov ecx, 256", - "238:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 238b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 41984", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 67584", + "add rsi, 43008", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 352", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 67584", - "mov ecx, 256", - "239:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 239b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "240:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 240b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "241:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 241b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1152", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "242:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 242b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 47104", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "243:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 243b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 49152", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "244:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 244b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 51200", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 672", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "245:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 245b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "246:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 246b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 77824", - "mov ecx, 256", - "247:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 247b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 10240", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1280", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 78848", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 78848", - "mov ecx, 256", - "248:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 248b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 55296", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 79872", - "mov ecx, 256", - "249:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 249b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 56320", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 80896", - "mov ecx, 256", - "250:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 250b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 57344", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 81920", - "mov ecx, 256", - "251:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 251b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 58368", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 82944", + "add rsi, 59392", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 992", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 82944", - "mov ecx, 256", - "252:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 252b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 83968", - "mov ecx, 256", - "253:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 253b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 86016", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 86016", - "mov ecx, 256", - "254:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 254b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 11264", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1408", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 87040", - "mov ecx, 256", - "255:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 255b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 62464", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 88064", + "add rsi, 63488", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 88064", - "mov ecx, 256", - "256:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 256b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 65536", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 89088", - "mov ecx, 256", - "257:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 257b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 90112", + "add rsi, 67584", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1312", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 90112", - "mov ecx, 256", - "258:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 258b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 91136", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 91136", - "mov ecx, 256", - "259:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 259b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 92160", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 92160", - "mov ecx, 256", - "260:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 260b", - "mov rdi, r13", - "add rdi, 0", - "mov esi, 64", - "mov edx, 60", - "mov rcx, rbx", - "add rcx, 23552", - "mov r8, rbx", - "add r8, 4096", - "call {vg_mldsa_sample_in_ball}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 23552", - "mov ecx, 256", - "261:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 261b", - "mov rdi, rbx", - "add rdi, 16384", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 17408", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 18432", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 19456", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 20480", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 21504", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 22528", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 23552", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 33792", - "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 34816", - "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 32", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", - "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 25600", - "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 23552", "mov rdx, rbx", "add rdx, 24576", "call {vg_mldsa_multiply_ntt_avx2}", @@ -7223,7 +6867,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 8192", + "add rdi, 12288", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7234,60 +6878,60 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1024", + "add rdx, 1536", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 69632", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 70656", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 71680", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 72704", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 74752", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 75776", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbp", - "add rdi, 352", + "add rdi, 1632", "mov rsi, rbx", "add rsi, 24576", "call {vg_mldsa_unpack_t1}", @@ -7314,7 +6958,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 9216", + "add rdi, 13312", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7325,60 +6969,60 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1152", + "add rdx, 1664", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 77824", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 78848", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 79872", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 80896", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 81920", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 82944", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 51200", + "add rsi, 83968", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbp", - "add rdi, 672", + "add rdi, 1952", "mov rsi, rbx", "add rsi, 24576", "call {vg_mldsa_unpack_t1}", @@ -7405,7 +7049,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 10240", + "add rdi, 14336", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7416,60 +7060,60 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1280", + "add rdx, 1792", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 86016", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 87040", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 88064", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 89088", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 90112", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 91136", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 92160", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbp", - "add rdi, 992", + "add rdi, 2272", "mov rsi, rbx", "add rsi, 24576", "call {vg_mldsa_unpack_t1}", @@ -7496,7 +7140,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 11264", + "add rdi, 15360", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7507,618 +7151,854 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1408", + "add rdx, 1920", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 61440", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 62464", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 63488", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 1024", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 64512", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 2048", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", "mov rsi, rbx", - "add rsi, 65536", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rsi, 2048", + "mov rdi, r13", + "add rdi, 0", + "mov ecx, 64", + "mov edx, 0", + "226:", + "movzx eax, BYTE PTR [rsi]", + "movzx r8d, BYTE PTR [rdi]", + "xor rax, r8", + "or rdx, rax", + "add rsi, 1", + "add rdi, 1", + "sub rcx, 1", + "jne 226b", + "sub rdx, 1", + "sbb rax, rax", + "and r15d, eax", + "23:", + "21:", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, + vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, + vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, + vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1, + vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + ) +} + +/// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address. +/// +/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails. +/// +/// # Safety +/// +/// * `seed` must be valid for reads of 32 bytes. +/// * `pk` must be valid for reads and writes of 2592 bytes. +/// * `sk` must be valid for reads and writes of 4896 bytes. +/// * `scratch` must be valid for reads and writes of 144384 bytes. +/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do). +/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk: *mut [u8; 2592], sk: *mut [u8; 4896], scratch: *mut [u64; 18048]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+840], rbx", + "mov QWORD PTR [rcx+848], rbp", + "mov QWORD PTR [rcx+856], r12", + "mov QWORD PTR [rcx+864], r13", + "mov QWORD PTR [rcx+872], r14", + "mov QWORD PTR [rcx+880], r15", + "mov rbx, rcx", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r15d, 1", + "mov eax, 8", + "mov BYTE PTR [rbx+896], al", + "mov eax, 7", + "mov BYTE PTR [rbx+897], al", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 66560", - "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 67584", - "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1312", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, rbx", + "add rcx, 896", + "mov r8d, 2", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 34", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 128", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", - "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rsi, 1024", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1216", "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rsi, 1056", + "mov ecx, 64", + "21:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 21b", + "mov eax, 0", + "mov BYTE PTR [rbx+1281], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 12288", - "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", - "mov rdi, rbx", - "add rdi, 27648", - "mov esi, 15", "mov rdx, rbx", - "add rdx, 1536", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 4096", + "mov ecx, 256", + "22:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 22b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 5120", "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 5120", + "mov ecx, 256", + "23:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 23b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 6144", "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 6144", + "mov ecx, 256", + "24:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 24b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 7168", "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 7168", + "mov ecx, 256", + "25:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 25b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 8192", + "mov ecx, 256", + "26:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 26b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 9216", "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 9216", + "mov ecx, 256", + "27:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 27b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 10240", "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 10240", + "mov ecx, 256", + "28:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 28b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 11264", "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1632", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "add rdi, 11264", + "mov ecx, 256", + "29:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 29b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rdi, 12288", + "mov ecx, 256", + "210:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 210b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 13312", - "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", - "mov rdi, rbx", - "add rdi, 27648", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 1664", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", + "mov ecx, 256", + "211:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 211b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 14336", "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 78848", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 14336", + "mov ecx, 256", + "212:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 212b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 15360", "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 80896", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 15360", + "mov ecx, 256", + "213:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 213b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 16384", "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 82944", - "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 16384", + "mov ecx, 256", + "214:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 214b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 17408", "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1952", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "add rdi, 17408", + "mov ecx, 256", + "215:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 215b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 18432", "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", + "add rdi, 18432", + "mov ecx, 256", + "216:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 216b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 14336", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", - "mov rdi, rbx", - "add rdi, 27648", - "mov esi, 15", + "add rsi, 19456", "mov rdx, rbx", - "add rdx, 1792", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 86016", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdi, 19456", + "mov ecx, 256", + "217:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 217b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 20480", "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 88064", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 20480", + "mov ecx, 256", + "218:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 218b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 21504", "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 90112", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 21504", + "mov ecx, 256", + "219:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 219b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 91136", + "add rsi, 22528", "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 22528", + "mov ecx, 256", + "220:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 220b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 92160", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 2272", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 23552", + "mov ecx, 256", + "221:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 221b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "mov ecx, 256", + "222:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 222b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 25600", "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 25600", + "mov ecx, 256", + "223:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 223b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rsi, 26624", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", + "mov ecx, 256", + "224:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 224b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 15360", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", + "add rsi, 27648", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 27648", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 1920", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", + "mov ecx, 256", + "225:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 225b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, r12", - "add rcx, 0", - "mov r8d, 64", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 64", - "mov rcx, rbx", - "add rcx, 1024", - "mov r8d, 1024", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov ecx, 31", - "mov r8, rbx", - "add r8, 200", - "call {vg_keccak_pad}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbx", - "add rcx, 2048", - "mov r8d, 64", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_squeeze}", - "mov rsi, rbx", - "add rsi, 2048", - "mov rdi, r13", - "add rdi, 0", - "mov ecx, 64", - "mov edx, 0", - "262:", - "movzx eax, BYTE PTR [rsi]", - "movzx r8d, BYTE PTR [rdi]", - "xor rax, r8", - "or rdx, rax", - "add rsi, 1", - "add rdi, 1", - "sub rcx, 1", - "jne 262b", - "sub rdx, 1", - "sbb rax, rax", - "and r15d, eax", - "23:", - "21:", - "mov eax, r15d", - "mov r15, QWORD PTR [rbx+880]", - "mov r14, QWORD PTR [rbx+872]", - "mov r13, QWORD PTR [rbx+864]", - "mov r12, QWORD PTR [rbx+856]", - "mov rbp, QWORD PTR [rbx+848]", - "mov rbx, QWORD PTR [rbx+840]", - "ret", - vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, - vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, - vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, - vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, - vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, - vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, - vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, - vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1, - vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, - vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, - vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint, - vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, - vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, - vg_keccak_pad = sym super::sha3::vg_keccak_pad, - vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, - ) -} - -/// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. -/// -/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. -/// -/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address. -/// -/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails. -/// -/// # Safety -/// -/// * `seed` must be valid for reads of 32 bytes. -/// * `pk` must be valid for reads and writes of 2592 bytes. -/// * `sk` must be valid for reads and writes of 4896 bytes. -/// * `scratch` must be valid for reads and writes of 144384 bytes. -/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before. -/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). -/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do). -/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). -#[unsafe(naked)] -pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk: *mut [u8; 2592], sk: *mut [u8; 4896], scratch: *mut [u64; 18048]) -> u32 { - core::arch::naked_asm!( - "mov QWORD PTR [rcx+840], rbx", - "mov QWORD PTR [rcx+848], rbp", - "mov QWORD PTR [rcx+856], r12", - "mov QWORD PTR [rcx+864], r13", - "mov QWORD PTR [rcx+872], r14", - "mov QWORD PTR [rcx+880], r15", - "mov rbx, rcx", - "mov rbp, rdi", - "mov r12, rsi", - "mov r13, rdx", - "mov r15d, 1", - "mov eax, 8", - "mov BYTE PTR [rbx+896], al", - "mov eax, 7", - "mov BYTE PTR [rbx+897], al", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbp", - "add rcx, 0", - "mov r8d, 32", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 32", - "mov rcx, rbx", - "add rcx, 896", - "mov r8d, 2", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 34", - "mov ecx, 31", - "mov r8, rbx", - "add r8, 200", - "call {vg_keccak_pad}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbx", - "add rcx, 1024", - "mov r8d, 128", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_squeeze}", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "20:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 20b", - "mov rdi, rbx", - "add rdi, 1216", - "mov rsi, rbx", - "add rsi, 1056", - "mov ecx, 64", - "21:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 21b", - "mov eax, 0", - "mov BYTE PTR [rbx+1281], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 28672", + "mov ecx, 256", + "226:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 226b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 4096", + "add rsi, 29696", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8126,23 +8006,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 4096", + "add rdi, 29696", "mov ecx, 256", - "22:", + "227:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 22b", - "mov eax, 1", + "jne 227b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 3", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 5120", + "add rsi, 30720", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8150,23 +8030,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 5120", + "add rdi, 30720", "mov ecx, 256", - "23:", + "228:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 23b", - "mov eax, 2", + "jne 228b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 3", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 6144", + "add rsi, 31744", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8174,23 +8054,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 6144", + "add rdi, 31744", "mov ecx, 256", - "24:", + "229:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 24b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "jne 229b", "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 7168", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8198,23 +8078,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 7168", + "add rdi, 32768", "mov ecx, 256", - "25:", + "230:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 25b", - "mov eax, 4", + "jne 230b", + "mov eax, 1", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8222,23 +8102,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 8192", + "add rdi, 33792", "mov ecx, 256", - "26:", + "231:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 26b", - "mov eax, 5", + "jne 231b", + "mov eax, 2", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 9216", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8246,23 +8126,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 9216", + "add rdi, 34816", "mov ecx, 256", - "27:", + "232:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 27b", - "mov eax, 6", + "jne 232b", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 10240", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8270,23 +8150,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 10240", + "add rdi, 35840", "mov ecx, 256", - "28:", + "233:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 28b", - "mov eax, 0", + "jne 233b", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8294,23 +8174,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 11264", + "add rdi, 36864", "mov ecx, 256", - "29:", + "234:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 29b", - "mov eax, 1", + "jne 234b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 12288", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8318,23 +8198,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 12288", + "add rdi, 37888", "mov ecx, 256", - "210:", + "235:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 210b", - "mov eax, 2", + "jne 235b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 13312", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8342,23 +8222,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 13312", + "add rdi, 38912", "mov ecx, 256", - "211:", + "236:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 211b", - "mov eax, 3", + "jne 236b", + "mov eax, 0", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 14336", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8366,23 +8246,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 14336", + "add rdi, 39936", "mov ecx, 256", - "212:", + "237:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 212b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", + "jne 237b", "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8390,23 +8270,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 15360", + "add rdi, 40960", "mov ecx, 256", - "213:", + "238:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 213b", - "mov eax, 5", + "jne 238b", + "mov eax, 2", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 16384", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8414,23 +8294,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 16384", + "add rdi, 41984", "mov ecx, 256", - "214:", + "239:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 214b", - "mov eax, 6", + "jne 239b", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 17408", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8438,23 +8318,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 17408", + "add rdi, 43008", "mov ecx, 256", - "215:", + "240:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 215b", - "mov eax, 0", + "jne 240b", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 18432", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8462,23 +8342,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 18432", + "add rdi, 44032", "mov ecx, 256", - "216:", + "241:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", - "mov eax, 1", + "jne 241b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8486,23 +8366,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 19456", + "add rdi, 45056", "mov ecx, 256", - "217:", + "242:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 217b", - "mov eax, 2", + "jne 242b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 20480", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8510,23 +8390,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 20480", + "add rdi, 46080", "mov ecx, 256", - "218:", + "243:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 218b", - "mov eax, 3", + "jne 243b", + "mov eax, 0", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 21504", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8534,23 +8414,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 21504", + "add rdi, 47104", "mov ecx, 256", - "219:", + "244:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 219b", - "mov eax, 4", + "jne 244b", + "mov eax, 1", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 22528", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8558,23 +8438,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 22528", + "add rdi, 48128", "mov ecx, 256", - "220:", + "245:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", + "jne 245b", "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8582,23 +8462,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 23552", + "add rdi, 49152", "mov ecx, 256", - "221:", + "246:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", - "mov eax, 6", + "jne 246b", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 24576", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8606,23 +8486,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", + "add rdi, 50176", "mov ecx, 256", - "222:", + "247:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", - "mov eax, 0", + "jne 247b", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 25600", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8630,23 +8510,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 51200", "mov ecx, 256", - "223:", + "248:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", - "mov eax, 1", + "jne 248b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 26624", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8654,23 +8534,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 52224", "mov ecx, 256", - "224:", + "249:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", - "mov eax, 2", + "jne 249b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 27648", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8678,23 +8558,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 27648", + "add rdi, 53248", "mov ecx, 256", - "225:", + "250:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", - "mov eax, 3", + "jne 250b", + "mov eax, 0", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 28672", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8702,23 +8582,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 28672", + "add rdi, 54272", "mov ecx, 256", - "226:", + "251:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 226b", - "mov eax, 4", + "jne 251b", + "mov eax, 1", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 29696", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8726,23 +8606,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 29696", + "add rdi, 55296", "mov ecx, 256", - "227:", + "252:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 227b", - "mov eax, 5", + "jne 252b", + "mov eax, 2", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 30720", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8750,23 +8630,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 30720", + "add rdi, 56320", "mov ecx, 256", - "228:", + "253:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 228b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", + "jne 253b", "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8774,23 +8654,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 31744", + "add rdi, 57344", "mov ecx, 256", - "229:", + "254:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 229b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "jne 254b", "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8798,23 +8678,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 32768", + "add rdi, 58368", "mov ecx, 256", - "230:", + "255:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 230b", - "mov eax, 1", + "jne 255b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 59392", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8822,23 +8702,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 59392", "mov ecx, 256", - "231:", + "256:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 231b", - "mov eax, 2", + "jne 256b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 60416", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8846,1197 +8726,759 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 60416", "mov ecx, 256", - "232:", + "257:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 232b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 257b", + "mov eax, 0", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 35840", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 61440", "mov ecx, 256", - "233:", + "258:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 233b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 258b", + "mov eax, 1", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 36864", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 62464", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 36864", + "add rdi, 62464", "mov ecx, 256", - "234:", + "259:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 234b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 259b", + "mov eax, 2", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 37888", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 63488", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 37888", + "add rdi, 63488", "mov ecx, 256", - "235:", + "260:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 235b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 260b", + "mov eax, 3", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 38912", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 64512", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 38912", + "add rdi, 64512", "mov ecx, 256", - "236:", + "261:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 236b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 261b", + "mov eax, 4", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 39936", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 65536", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 39936", + "add rdi, 65536", "mov ecx, 256", - "237:", + "262:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 237b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "jne 262b", "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 40960", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 66560", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 40960", + "add rdi, 66560", "mov ecx, 256", - "238:", + "263:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 238b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 263b", + "mov eax, 6", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 41984", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 67584", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 41984", + "add rdi, 67584", "mov ecx, 256", - "239:", + "264:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 239b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 264b", + "mov eax, 7", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 43008", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 68608", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 43008", + "add rdi, 68608", "mov ecx, 256", - "240:", + "265:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 240b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 265b", + "mov eax, 8", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 44032", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 69632", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 44032", + "add rdi, 69632", "mov ecx, 256", - "241:", + "266:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 241b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 266b", + "mov eax, 9", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 45056", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 70656", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 45056", + "add rdi, 70656", "mov ecx, 256", - "242:", + "267:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 242b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 267b", + "mov eax, 10", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 46080", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 71680", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 46080", + "add rdi, 71680", "mov ecx, 256", - "243:", + "268:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 243b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 268b", + "mov eax, 11", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 47104", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 72704", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 47104", + "add rdi, 72704", "mov ecx, 256", - "244:", + "269:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 244b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 269b", + "mov eax, 12", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 48128", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 73728", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 48128", + "add rdi, 73728", "mov ecx, 256", - "245:", + "270:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 245b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 270b", + "mov eax, 13", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 49152", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 74752", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 49152", + "add rdi, 74752", "mov ecx, 256", - "246:", + "271:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 246b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 271b", + "mov eax, 14", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 50176", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 75776", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 50176", + "add rdi, 75776", "mov ecx, 256", - "247:", + "272:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 247b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "jne 272b", + "mov rdi, r12", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 51200", - "mov ecx, 256", - "248:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "273:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 248b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "jne 273b", + "mov rdi, r13", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 52224", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 52224", - "mov ecx, 256", - "249:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "274:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 249b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "jne 274b", + "mov rdi, r13", + "add rdi, 32", "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "250:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1120", + "mov ecx, 32", + "275:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 250b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "jne 275b", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 61440", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 128", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "251:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 251b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 62464", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 224", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 63488", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 320", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "252:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 252b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 64512", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 416", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 65536", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 512", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "253:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 253b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 66560", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 608", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 67584", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 704", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "254:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 254b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 68608", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 800", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 58368", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 69632", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 896", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "255:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 255b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 70656", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 992", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 71680", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1088", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "256:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 256b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 72704", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1184", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 60416", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 73728", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1280", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 60416", - "mov ecx, 256", - "257:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 257b", - "mov eax, 0", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 74752", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1376", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", + "add rdi, 75776", "mov esi, 2", - "mov rdx, rbx", - "add rdx, 61440", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1472", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 61440", - "mov ecx, 256", - "258:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 258b", - "mov eax, 1", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 62464", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 62464", - "mov ecx, 256", - "259:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 259b", - "mov eax, 2", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 63488", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 63488", - "mov ecx, 256", - "260:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 260b", - "mov eax, 3", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 64512", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 64512", - "mov ecx, 256", - "261:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 261b", - "mov eax, 4", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 65536", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 65536", - "mov ecx, 256", - "262:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 262b", - "mov eax, 5", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 66560", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 66560", - "mov ecx, 256", - "263:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 263b", - "mov eax, 6", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 67584", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 67584", - "mov ecx, 256", - "264:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 264b", - "mov eax, 7", - "mov BYTE PTR [rbx+1280], al", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 4096", "mov rdx, rbx", - "add rdx, 68608", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", - "add rdi, 68608", - "mov ecx, 256", - "265:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 265b", - "mov eax, 8", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 6144", "mov rdx, rbx", - "add rdx, 69632", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "266:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 266b", - "mov eax, 9", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 7168", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 70656", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "267:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 267b", - "mov eax, 10", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 9216", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 10240", "mov rdx, rbx", - "add rdx, 71680", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "268:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 268b", - "mov eax, 11", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 68608", + "call {vg_mldsa_add}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 72704", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "269:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 269b", - "mov eax, 12", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 32", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 73728", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1568", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "270:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 270b", - "mov eax, 13", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 11264", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 74752", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "271:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 271b", - "mov eax, 14", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 14336", "mov rdx, rbx", - "add rdx, 75776", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "272:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 272b", - "mov rdi, r12", - "add rdi, 0", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "273:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 273b", - "mov rdi, r13", - "add rdi, 0", + "add rsi, 15360", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "274:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 274b", - "mov rdi, r13", - "add rdi, 32", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1120", - "mov ecx, 32", - "275:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 275b", + "add rsi, 17408", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 61440", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 128", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt}", "mov rdi, rbx", - "add rdi, 62464", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 224", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 69632", + "call {vg_mldsa_add}", "mov rdi, rbx", - "add rdi, 63488", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 320", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 64512", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 416", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 352", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 65536", - "mov esi, 2", - "mov edx, 2", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", "mov rcx, r13", - "add rcx, 512", - "mov r8d, 96", + "add rcx, 1984", + "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 66560", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 608", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 67584", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 704", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 68608", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 800", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 69632", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 896", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 70656", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 992", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 71680", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1088", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 72704", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1184", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 73728", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1280", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 74752", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1376", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 75776", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1472", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 61440", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 62464", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 63488", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 64512", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 65536", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 66560", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 67584", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 18432", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 5120", + "add rsi, 19456", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 6144", + "add rsi, 20480", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 7168", + "add rsi, 21504", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 22528", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 9216", + "add rsi, 23552", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 10240", + "add rsi, 24576", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10048,7 +9490,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 68608", + "add rsi, 70656", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10061,7 +9503,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 32", + "add rdx, 672", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10069,55 +9511,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 1568", + "add rcx, 2400", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 25600", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 12288", + "add rsi, 26624", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 13312", + "add rsi, 27648", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 14336", + "add rsi, 28672", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 29696", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 16384", + "add rsi, 30720", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 17408", + "add rsi, 31744", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10129,7 +9571,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 71680", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10142,7 +9584,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 352", + "add rdx, 992", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10150,55 +9592,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 1984", + "add rcx, 2816", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 18432", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 20480", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 21504", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 22528", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 24576", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10210,7 +9652,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 72704", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10223,7 +9665,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 672", + "add rdx, 1312", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10231,55 +9673,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 2400", + "add rcx, 3232", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 25600", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 26624", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 27648", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 28672", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 29696", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 30720", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10291,7 +9733,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 73728", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10304,7 +9746,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 992", + "add rdx, 1632", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10312,55 +9754,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 2816", + "add rcx, 3648", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 35840", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10372,7 +9814,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 74752", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10385,7 +9827,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 1312", + "add rdx, 1952", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10393,55 +9835,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 3232", + "add rcx, 4064", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 44032", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 59392", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 60416", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10453,7 +9895,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 75776", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10466,7 +9908,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 1632", + "add rdx, 2272", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10474,197 +9916,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 3648", + "add rcx, 4480", "mov r8d, 416", "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 50176", - "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 52224", - "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 74752", - "call {vg_mldsa_add}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 1952", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 4064", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 58368", - "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 60416", - "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 75776", - "call {vg_mldsa_add}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 2272", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 4480", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", "add rdi, 0", "mov esi, 136", @@ -13576,7 +12856,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: /// * `scratch` must be valid for reads and writes of 144384 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu: *const [u8; 64], sig: *const [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { core::arch::naked_asm!( @@ -13636,1425 +12916,705 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 524287", "mov ecx, 524288", "mov r8, rbx", - "add r8, 18432", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 18432", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 1984", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 19456", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 19456", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 2624", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 20480", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 20480", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 3264", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 21504", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 21504", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 3904", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 22528", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 22528", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "test r15d, r15d", - "jne 22f", - "jmp 23f", - "22:", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbp", - "add rsi, 0", - "mov ecx, 32", - "24:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", - "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", - "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", - "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", - "29:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 29b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 33792", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 33792", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 34816", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 34816", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 36864", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 39936", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 40960", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 41984", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 41984", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 43008", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 43008", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "220:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 220b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "221:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 221b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "222:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 222b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", - "223:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 223b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 50176", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 50176", - "mov ecx, 256", - "224:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 224b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 51200", - "mov ecx, 256", - "225:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 225b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 58368", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 61440", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 62464", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 63488", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 64512", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "236:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 236b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 65536", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "237:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 237b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 66560", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 66560", - "mov ecx, 256", - "238:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 238b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 67584", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 67584", - "mov ecx, 256", - "239:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 239b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add r8, 18432", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 69632", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdi, 18432", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "mov rdi, r13", + "add rdi, 1984", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 19456", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "240:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 240b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 19456", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 2624", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 20480", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdi, 20480", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "mov rdi, r13", + "add rdi, 3264", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 21504", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "241:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 241b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 21504", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 3904", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 22528", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 71680", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdi, 22528", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "test r15d, r15d", + "jne 22f", + "jmp 23f", + "22:", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "242:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rdi, 896", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 242b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "jne 24b", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 72704", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "25:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 25b", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "243:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "26:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 243b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "jne 26b", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 73728", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "27:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 27b", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "244:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "28:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 244b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "jne 28b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "245:", + "add rdi, 28672", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 245b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", + "jne 29b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 31744", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "246:", + "add rdi, 31744", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 246b", + "jne 210b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 77824", - "mov ecx, 256", - "247:", + "add rdi, 36864", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 247b", + "jne 211b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 78848", + "add rsi, 39936", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 78848", - "mov ecx, 256", - "248:", + "add rdi, 39936", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 248b", + "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 79872", - "mov ecx, 256", - "249:", + "add rdi, 45056", + "mov ecx, 1024", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 249b", + "jne 213b", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 48128", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 80896", - "mov ecx, 256", - "250:", + "add rdi, 48128", + "mov ecx, 1024", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 250b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 81920", - "mov ecx, 256", - "251:", + "add rdi, 53248", + "mov ecx, 1024", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 251b", + "jne 215b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 82944", + "add rsi, 56320", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 82944", - "mov ecx, 256", - "252:", + "add rdi, 56320", + "mov ecx, 1024", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 252b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 83968", - "mov ecx, 256", - "253:", + "add rdi, 61440", + "mov ecx, 1024", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 253b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "jne 217b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 86016", + "add rsi, 64512", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 86016", - "mov ecx, 256", - "254:", + "add rdi, 64512", + "mov ecx, 1024", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 254b", + "jne 218b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 87040", - "mov ecx, 256", - "255:", + "add rdi, 69632", + "mov ecx, 1024", + "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 255b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "jne 219b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 88064", + "add rsi, 72704", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 88064", - "mov ecx, 256", - "256:", + "add rdi, 72704", + "mov ecx, 1024", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 256b", + "jne 220b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 89088", - "mov ecx, 256", - "257:", + "add rdi, 77824", + "mov ecx, 1024", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 257b", + "jne 221b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 90112", + "add rsi, 80896", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 90112", - "mov ecx, 256", - "258:", + "add rdi, 80896", + "mov ecx, 1024", + "222:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 258b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "jne 222b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 91136", + "add rsi, 86016", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 91136", - "mov ecx, 256", - "259:", + "add rdi, 86016", + "mov ecx, 1024", + "223:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 259b", + "jne 223b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 92160", + "add rsi, 89088", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 92160", - "mov ecx, 256", - "260:", + "add rdi, 89088", + "mov ecx, 1024", + "224:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 260b", + "jne 224b", "mov rdi, r13", "add rdi, 0", "mov esi, 64", @@ -15070,13 +13630,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "261:", + "225:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 261b", + "jne 225b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -15915,7 +14475,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rdi, 0", "mov ecx, 64", "mov edx, 0", - "262:", + "226:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -15923,7 +14483,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 262b", + "jne 226b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -15940,7 +14500,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt,