diff --git a/README.md b/README.md index 02ab01664..a4315a7c3 100644 --- a/README.md +++ b/README.md @@ -827,7 +827,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) ✅ SHA extensions @@ -843,7 +843,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) ✅ SHA extensions @@ -859,7 +859,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2) ✅ SHA extensions diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml index 766d606c3..4f28100c0 100644 --- a/docs/algorithms/ml-dsa-44.toml +++ b/docs/algorithms/ml-dsa-44.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa44.rs"] asm = ["mldsa44", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" } diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml index 8835d083b..6683e2898 100644 --- a/docs/algorithms/ml-dsa-65.toml +++ b/docs/algorithms/ml-dsa-65.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa65.rs"] asm = ["mldsa65", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" } diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml index 9a82d4299..649298bea 100644 --- a/docs/algorithms/ml-dsa-87.toml +++ b/docs/algorithms/ml-dsa-87.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa87.rs"] asm = ["mldsa87", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; matrix sampled four entries at a time in verification (four SHAKE128 instances at once with AVX2)" } diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean index ca47fca64..67b44002a 100644 --- a/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean +++ b/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean @@ -4,6 +4,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejNttCT import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejBoundedCT import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.ExpandMask import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.BallCT +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Verified /-! # ML-DSA (FIPS 204) on x86-64: the sampling primitives @@ -32,6 +33,28 @@ def artifacts : List Artifact := [ stack := 16 verified := Proof.MlDsa.X86_64.Sample.rejNTT_verified spSafe := Code.all_of_allInstrs (by lit_decide) }, + { Spec.MlDsa.rejNTT4Api with + target := X86_64.target + doc := Spec.MlDsa.rejNTT4Api.doc (notes := ["It calls `vg_mldsa_rej_ntt_poly` on each seed, and saves \ + its caller's callee-saved registers in `scratch`."]) + code := Impl.MlDsa.X86_64.Sample.Rej4.rejNTT4 + contract := Spec.MlDsa.rejNTT4Contract X86_64.abi 24 + stack := 24 + verified := Proof.MlDsa.X86_64.Rej4.rejNTT4_verified + spSafe := Code.all_of_allInstrs (by decide +kernel) }, + { Spec.MlDsa.rejNTT4Api with + name := Spec.MlDsa.rejNTT4Api.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.rejNTT4Api.doc (notes := ["It runs the four instances of SHAKE128 at once, in the four \ + 64-bit elements of AVX2 registers (as `vg_mlkem_sample_ntt4_avx2` does), squeezing three blocks of each \ + twice, and runs the loop of `RejNTTPoly` over the 1008 bytes of each seed's output, as \ + `vg_mldsa_rej_ntt_poly` does. It saves its caller's callee-saved registers in `scratch`."]) + code := Impl.MlDsa.X86_64.Sample.Rej4.rejNTT4Avx2 + contract := Spec.MlDsa.rejNTT4Contract X86_64.abi 24 + stack := 24 + verified := Proof.MlDsa.X86_64.Rej4.rejNTT4Avx2_verified + spSafe := Code.all_of_allInstrs (by decide +kernel) + features := ["avx", "avx2"] }, { Spec.MlDsa.rejBoundedApi with target := X86_64.target doc := Spec.MlDsa.rejBoundedApi.doc (notes := ["It squeezes 544 bytes of SHAKE256 output (4 blocks) and \ diff --git a/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean index 903d1f596..ec57e7834 100644 --- a/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean +++ b/lean/VerifiedGarbage/Generic/MlDsaArith/X86_64/MlDsaVerify.lean @@ -16,8 +16,8 @@ emitter adds the `# Safety` items that depend on the target (`Sig.layoutDoc`), from `stack` and `writeArgs`, which `ofSig` checks against the contract. -The stack is 24 bytes: the return address of a call of a primitive, and up -to 16 bytes for its own calls. +The stack is 32 bytes: the return address of a call of a primitive, and up +to 24 bytes for its own calls (`vg_mldsa_rej_ntt_poly4`'s). -/ namespace VG.Generic.MlDsaArith.X86_64.MlDsaVerify @@ -38,8 +38,8 @@ def artifacts (v : ArithImpl) : List Artifact := [ target := X86_64.target doc := Spec.MlDsa.verify44Api.doc (notes := [note]) code := Impl.MlDsa.X86_64.Verify.verify (primsWith v.code) Spec.MlDsa.mlDsa44 - contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa44 X86_64.abi 24 - stack := 24 + contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa44 X86_64.abi 32 + stack := 32 verified := verify_prims v (List.mem_cons_self ..) spSafe := verify_spSafe (prims_okWith v) (List.mem_cons_self ..) }, { Spec.MlDsa.verify65Api with @@ -48,8 +48,8 @@ def artifacts (v : ArithImpl) : List Artifact := [ target := X86_64.target doc := Spec.MlDsa.verify65Api.doc (notes := [note]) code := Impl.MlDsa.X86_64.Verify.verify (primsWith v.code) Spec.MlDsa.mlDsa65 - contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa65 X86_64.abi 24 - stack := 24 + contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa65 X86_64.abi 32 + stack := 32 verified := verify_prims v (List.mem_cons_of_mem _ (List.mem_cons_self ..)) spSafe := verify_spSafe (prims_okWith v) (List.mem_cons_of_mem _ (List.mem_cons_self ..)) }, { Spec.MlDsa.verify87Api with @@ -58,8 +58,8 @@ def artifacts (v : ArithImpl) : List Artifact := [ target := X86_64.target doc := Spec.MlDsa.verify87Api.doc (notes := [note]) code := Impl.MlDsa.X86_64.Verify.verify (primsWith v.code) Spec.MlDsa.mlDsa87 - contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa87 X86_64.abi 24 - stack := 24 + contract := Spec.MlDsa.verifyContract Spec.MlDsa.mlDsa87 X86_64.abi 32 + stack := 32 verified := verify_prims v (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))) spSafe := verify_spSafe (prims_okWith v) (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))) }] diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean index 4c4172ca7..3c987cbb6 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean @@ -198,6 +198,7 @@ def subAvx2 : Prog isa := .seq (.block (yconst .xmm15 8380417)) (.seq (rcxLoop 32 (yaccBody .psubd (vcadd .xmm0 .xmm2))) (.block yepi)) /-- The AVX2 code. -/ -def Backend.avx2 : Backend := ⟨nttAvx2, nttInvAvx2, mulAvx2, mulAddAvx2, addAvx2, subAvx2, "_avx2"⟩ +def Backend.avx2 : Backend := + ⟨nttAvx2, nttInvAvx2, mulAvx2, mulAddAvx2, addAvx2, subAvx2, Sample.Rej4.rejNTT4Avx2, "_avx2"⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean index c6ef074df..65d34d28d 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean @@ -1,6 +1,7 @@ import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Ntt import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Mul import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.AddSub +import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejNtt4 /-! # ML-DSA on x86-64: implementations of the polynomial arithmetic @@ -8,7 +9,8 @@ import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.AddSub Key generation, signing and verification call the polynomial arithmetic of one implementation, a `Backend`: the code of `vg_mldsa_ntt`, `vg_mldsa_inv_ntt`, `vg_mldsa_multiply_ntt`, `vg_mldsa_multiply_add_ntt`, -`vg_mldsa_add` and `vg_mldsa_sub`, whose names end with `sfx` (e.g. +`vg_mldsa_add` and `vg_mldsa_sub`, and of `vg_mldsa_rej_ntt_poly4` +(which samples four entries of the matrix `Â` at once), whose names end with `sfx` (e.g. `_avx2`; nothing for the SSE2 code, `sse2`). Each is a variant of the interface `MlDsaArith` on x86-64 (`Variants/MlDsaArith/X86_64/`), and the functions that call them are emitted once for each @@ -27,15 +29,17 @@ structure Backend where mulAdd : Prog isa add : Prog isa sub : Prog isa + rej4 : Prog isa /-- What the names of its functions, and of those calling them, end with. -/ sfx : String /-- The SSE2 code. -/ -def Backend.sse2 : Backend := ⟨Arith.ntt, Arith.nttInv, Arith.mul, Arith.mulAdd, Arith.add, Arith.sub, ""⟩ +def Backend.sse2 : Backend := + ⟨Arith.ntt, Arith.nttInv, Arith.mul, Arith.mulAdd, Arith.add, Arith.sub, Sample.Rej4.rejNTT4, ""⟩ /-- Every function empty, which the proofs that the functions calling a backend never write `rsp` (and load MXCSR only to restore it) evaluate in its place. -/ -def Backend.empty : Backend := ⟨.block [], .block [], .block [], .block [], .block [], .block [], ""⟩ +def Backend.empty : Backend := ⟨.block [], .block [], .block [], .block [], .block [], .block [], .block [], ""⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/RejNtt4.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/RejNtt4.lean new file mode 100644 index 000000000..a13540e9d --- /dev/null +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/RejNtt4.lean @@ -0,0 +1,85 @@ +import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejNtt +import VerifiedGarbage.Impl.MlKem.X86_64.Sample4 + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4` and `vg_mldsa_rej_ntt_poly4_avx2` + +`rejNTT4(seeds = rdi, a = rsi, scratch = rdx) -> eax` runs `RejNTTPoly` on +four seeds. The baseline implementation (`rejNTT4`) calls +`vg_mldsa_rej_ntt_poly` on each, with the prologue and epilogue of +`vg_mlkem_sample_ntt4` (`Impl/MlKem/X86_64/Sample4.lean`: `scratch` in +`rbx`, `seeds` in `r12`, `a` in `r13`, the AND of the results in `r14`, and +their caller's values and `rbp`'s saved in `scratch[4384..4424)`) and its +scratch space from byte 6144 of `scratch`. + +The one for AVX2 (`rejNTT4Avx2`) runs the four SHAKE128 instances at once +in the four 64-bit elements of `ymm` registers, as `vg_mlkem_sample_ntt4_avx2` +does, with the same code to absorb the seeds and squeeze three blocks of +each (504 bytes, from byte `2368 + 504 k` of `scratch`). It samples from them +with the loop of `vg_mldsa_rej_ntt_poly` (`rnBody`, 168 iterations of 3 +bytes), keeping the number `j` of coefficients of seed `k` at +`scratch[4424 + 8 k]`; squeezes three more blocks of each to the same +place, and runs 168 more iterations of the loop on them. That is the loop of +`vg_mldsa_rej_ntt_poly` on the same 1008 bytes of output, so seed `k` has +256 coefficients if and only if `vg_mldsa_rej_ntt_poly` would have them +(and if not, neither has `RejNTTPoly` within the least bound of FIPS 204 +Appendix C, 894 bytes). It returns 1 if every seed has 256 coefficients +(the AND of `j >> 8`), and 0 otherwise. + +The loops' branches and the addresses of their stores depend on the XOF +output, a function of the seeds, and on nothing else; every other address +and branch depends only on the pointers. +-/ + +namespace VG.Impl.MlDsa.X86_64.Sample.Rej4 + +open VG.X86_64 +open VG.Impl.MlKem.X86_64 (at_) +open VG.Impl.MlKem.X86_64.Sample4 (absorb4 squeeze4 oRc oBuf oScalar) +open VG.Impl.Sha3.X86_64.X4 (rcTable) + +/-- Where `j` of seed `k` is kept, at `scratch + oJ + 8 k`. -/ +def oJ : Nat := 4424 + +/-- `j ← 0` for each seed. -/ +def zeroJ : List Instr := + .mov32 .rax (.imm 0) :: (List.range 4).map fun k => .store (at_ .rbx (oJ + 8 * k)) .rax + +/-- 168 iterations of `RejNTTPoly`'s loop on the 504 bytes of the output of +seed `k`, to polynomial `k`, from `j` at `scratch + oJ + 8 k`. -/ +def half (k : Nat) : Prog isa := + .seq (.block [.mov .rsi (.reg .rbx), .alu .add .rsi (.imm (BitVec.ofNat 32 (oBuf + 504 * k))), + .mov .rbp (.reg .r13), .alu .add .rbp (.imm (BitVec.ofNat 32 (1024 * k))), + .mov .rdi (.mem (at_ .rbx (oJ + 8 * k))), .mov32 .rcx (.imm 168)]) + (.loop rnBody .ne) + +/-- The first half of seed `k`, and `j` kept. -/ +def first (k : Nat) : Prog isa := .seq (half k) (.block [.store (at_ .rbx (oJ + 8 * k)) .rdi]) + +/-- The second half of seed `k`, and `r14 ← r14 ∧ (j >> 8)`. -/ +def second (k : Nat) : Prog isa := + .seq (half k) (.block [.mov .rax (.reg .rdi), .shift .shr .rax 8, .alu32 .and .r14 (.reg .rax)]) + +/-- `vg_mldsa_rej_ntt_poly4_avx2`. `vzeroupper` clears the upper halves of +the vector registers after the last permutation. -/ +def rejNTT4Avx2 : Prog isa := + .seq (.block (VG.Impl.MlKem.X86_64.Sample4.pro ++ rcTable .rbx (oRc / 32) ++ absorb4)) + (.seq (squeeze4 0) (.seq (squeeze4 1) (.seq (squeeze4 2) (.seq (.block zeroJ) + (.seq (first 0) (.seq (first 1) (.seq (first 2) (.seq (first 3) + (.seq (squeeze4 0) (.seq (squeeze4 1) (.seq (squeeze4 2) (.seq (.block [.vop .vzeroupper]) + (.seq (second 0) (.seq (second 1) (.seq (second 2) (.seq (second 3) (.block VG.Impl.MlKem.X86_64.Sample4.epi))))))))))))))))) + +/-- `vg_mldsa_rej_ntt_poly` on seed `k`, and `r14 ← r14 ∧ result`. -/ +def callK (k : Nat) : Prog isa := + .seq (.block [.mov .rdi (.reg .r12), .alu .add .rdi (.imm (BitVec.ofNat 32 (34 * k))), + .mov .rsi (.reg .r13), .alu .add .rsi (.imm (BitVec.ofNat 32 (1024 * k))), .mov .rdx (.reg .rbx), + .alu .add .rdx (.imm (BitVec.ofNat 32 oScalar))]) + (.seq (.call "vg_mldsa_rej_ntt_poly" rejNTT) (.block [.alu32 .and .r14 (.reg .rax)])) + +/-- `vg_mldsa_rej_ntt_poly4`: `vg_mldsa_rej_ntt_poly` on each seed, with the +prologue and epilogue of `vg_mldsa_rej_ntt_poly4_avx2`. -/ +def rejNTT4 : Prog isa := + .seq (.block VG.Impl.MlKem.X86_64.Sample4.pro) (.seq (callK 0) (.seq (callK 1) (.seq (callK 2) (.seq (callK 3) + (.block VG.Impl.MlKem.X86_64.Sample4.epi))))) + +end VG.Impl.MlDsa.X86_64.Sample.Rej4 diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean index 5b4e5e24f..72ef6c175 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean @@ -21,11 +21,13 @@ The layout of `scratch` (in bytes): the Keccak state at 0 (200 bytes) and the sponge functions' working space at 200 (640 bytes); the caller's callee-saved registers at 840 (48 bytes); the seed of `RejNTTPoly` (`SB`, 34 bytes) at 896; `w1Encode(w′₁)` at 1024 (at most 1024 bytes); the -recomputed commitment hash `c̃′` at 2048 (at most 64 bytes); the working +recomputed commitment hash `c̃′` at 2048 (at most 64 bytes); the four seeds +of `vg_mldsa_rej_ntt_poly4` (`SB4`, 136 bytes) at 2560; the working space of the primitives at 4096 (2048 bytes); and polynomials of 1024 bytes from 8192 (`P j`): the hint `h` (polynomials 0 to 7, of which the first `k`), `z` (8 to 14), `c` (15), two temporaries (16, 17), `w′` (18), `w′₁` -(19), and `Â[r, s]` (`20 + 8r + s`). +(19), and `Â[r, s]` (`20 + 8r + s`), then the working space of +`vg_mldsa_rej_ntt_poly4` (8 KiB, after the last row of `Â`). -/ namespace VG.Impl.MlDsa.X86_64.Verify @@ -42,6 +44,7 @@ def oSV : Nat := 840 def oSB : Nat := 896 def oB : Nat := 1024 def oCT : Nat := 2048 +def oSB4 : Nat := 2560 def oSS : Nat := 4096 /-- Polynomial `j`. -/ def oP (j : Nat) : Nat := 8192 + 1024 * j @@ -137,6 +140,7 @@ structure Prims where unpackT1 : Prog isa hintUnpack : Prog isa normLt : Prog isa + rej4 : Prog isa /-- What the names of the polynomial arithmetic's functions end with (`Arith.Backend`). -/ sfx : String := "" @@ -159,6 +163,9 @@ def subAt (f g : Ptr) : Prog isa := callAt ("vg_mldsa_sub" ++ P.sfx) P.sub [(.rd def rejNttAt (a : Ptr) : Prog isa := callAt "vg_mldsa_rej_ntt_poly" P.rejNtt [(.rdi, .ptr (sc oSB)), (.rsi, .ptr a), (.rdx, .ptr (sc oSS))] +def rej4At (a w : Ptr) : Prog isa := + callAt ("vg_mldsa_rej_ntt_poly4" ++ P.sfx) P.rej4 [(.rdi, .ptr (sc oSB4)), (.rsi, .ptr a), (.rdx, .ptr w)] + def ballAt (ct : Ptr) (len tau : Nat) (c : Ptr) : Prog isa := callAt "vg_mldsa_sample_in_ball" P.ball [(.rdi, .ptr ct), (.rsi, .imm len), (.rdx, .imm tau), (.rcx, .ptr c), (.r8, .ptr (sc oSS))] @@ -191,12 +198,13 @@ end /-- `r15 ← r15 ∧ eax`. -/ def and15 : List Instr := [.alu32 .and .r15 (.reg .rax)] -/-- The polynomial at `a` masked by the result `eax` (0 or 1) of the -sampler that wrote it: unchanged if 1, and zero if 0, so that it is reduced -either way, without a branch. `edx ← -eax`, then each coefficient `∧ edx`. -/ -def mask (a : Ptr) : Prog isa := +/-- The polynomial at `a` (or the `N` coefficients from `a`) masked by the +result `eax` (0 or 1) of the sampler that wrote it: unchanged if 1, and zero +if 0, so that it is reduced either way, without a branch. `edx ← -eax`, then +each coefficient `∧ edx`. -/ +def mask (a : Ptr) (N : Nat := 256) : Prog isa := .seq (.block ([.mov32 .rdx (.imm 0), .alu32 .sub .rdx (.reg .rax)] ++ - glue [(.rdi, .ptr a), (.rcx, .imm 256)])) + glue [(.rdi, .ptr a), (.rcx, .imm N)])) (.loop (.block [.mov32 .rax (.mem (at_ .rdi 0)), .alu32 .and .rax (.reg .rdx), .store32 (at_ .rdi 0) .rax, .alu .add .rdi (.imm 4), .alu .sub .rcx (.imm 1)]) .ne) @@ -204,6 +212,10 @@ def mask (a : Ptr) : Prog isa := def sampled (call : Prog isa) (a : Ptr) : Prog isa := .seq call (.seq (.block and15) (mask a)) +/-- The same for a call that samples the four polynomials from `a`. -/ +def sampled4 (call : Prog isa) (a : Ptr) : Prog isa := + .seq call (.seq (.block and15) (mask a 1024)) + /-- `c` if `r15 ≠ 0`. -/ def ifOk (c : Prog isa) : Prog isa := .seq (.block [.alu32 .test .r15 (.reg .r15)]) (.ite .ne c (.block [])) diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean index b985e6cf0..4ef3f67af 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean @@ -16,8 +16,11 @@ in `scratch`. returns 0 at once if the hint is malformed. 2. `z[i] = BitUnpack` of the `i`-th piece of `σ` (polynomial `8 + i`), and `r15 ← r15 ∧ (‖z[i]‖∞ < γ₁ - β)`; it returns 0 if one of them is not. -3. `ρ` (`pk[0 : 32]`) to `SB`, and `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)` - (polynomial `20 + 8r + s`); `c = SampleInBall(c̃)` (polynomial 15). Each +3. `ρ` (`pk[0 : 32]`) to `SB` and to each seed of `SB4`, and + `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)` (polynomial `20 + 8r + s`), four + entries of a row at a time (`vg_mldsa_rej_ntt_poly4`): those from 0, then + for `ℓ = 7` those from 3 (sampling entry 3 again), or for `ℓ = 5` entry 4 + alone; `c = SampleInBall(c̃)` (polynomial 15). Each sampler's result is ANDed into `r15`, and its output masked with it (`sampled`): a sampler that fails leaves its output unspecified, and masking makes it reduced (zero) without a branch on the result, which @@ -73,12 +76,29 @@ def aOne (e : Nat) : Prog isa := .seq (.block (setB (sc (oSB + 32)) (e % 8) ++ setB (sc (oSB + 33)) (e / 8))) (sampled (rejNttAt P (pS (20 + e))) (pS (20 + e))) -/-- The entries `8r + s` of row `r` of `Â`. -/ -def aRow (r : Nat) : Prog isa := seqR (aOne P) (8 * r) p.ℓ +/-- Where `vg_mldsa_rej_ntt_poly4` works: after the last row of `Â`. -/ +def oR4 : Nat := oP (20 + 8 * p.k) -/-- `ρ` to `SB`, `Â`, and `c`. -/ +/-- The bytes `s ‖ r` of seed `k` of `SB4`. -/ +def setSR (r s k : Nat) : List Instr := setB (sc (oSB4 + 34 * k + 32)) (s + k) ++ setB (sc (oSB4 + 34 * k + 33)) r + +/-- `Â[r, s], …, Â[r, s + 3]`. -/ +def aGrp (r s : Nat) : Prog isa := + .seq (.block (setSR r s 0)) (.seq (.block (setSR r s 1)) (.seq (.block (setSR r s 2)) (.seq (.block (setSR r s 3)) + (sampled4 (rej4At P (pA r s) (sc (oR4 p))) (pA r s))))) + +/-- The entries of row `r` of `Â`: four from 0, then the last four if `ℓ = 7`, or the last one if `ℓ = 5`. -/ +def aRow (r : Nat) : Prog isa := + .seq (aGrp P p r 0) (if p.ℓ = 7 then aGrp P p r 3 else seqR (aOne P) (8 * r + 4) (p.ℓ - 4)) + +/-- `ρ` to `SB` and to the four seeds of `SB4`. -/ +def rhos : Prog isa := + .seq (copy (sc oSB) (.rbp, 0) 32) (.seq (copy (sc oSB4) (.rbp, 0) 32) (.seq (copy (sc (oSB4 + 34)) (.rbp, 0) 32) + (.seq (copy (sc (oSB4 + 68)) (.rbp, 0) 32) (copy (sc (oSB4 + 102)) (.rbp, 0) 32)))) + +/-- `ρ` to `SB` and `SB4`, `Â`, and `c`. -/ def samples : Prog isa := - .seq (copy (sc oSB) (.rbp, 0) 32) (.seq (seqR (aRow P p) 0 p.k) + .seq rhos (.seq (seqR (aRow P p) 0 p.k) (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC)) /-- `Σₛ Â[r, s] ẑ[s]` to `W`. -/ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean index 6a27d6c37..d8ad5b2bc 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean @@ -4,6 +4,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub import VerifiedGarbage.Proof.MlKem.X86_64.ArithOk +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Verified /-! # ML-DSA on x86-64: what the callers of the polynomial arithmetic need of it @@ -31,6 +32,15 @@ structure FnOk (k : Nat → Contract isa) (c : Prog isa) : Prop where ctl : ctlOk c = true sp : c.all (fun i => !isa.writesSp i) = true +/-- What a caller needs of `vg_mldsa_rej_ntt_poly4`, which calls three deep +and takes 24 bytes of stack. -/ +structure Rej4Ok (c : Prog isa) : Prop where + ver : Verified X86_64.target c (Spec.MlDsa.rejNTT4Contract X86_64.abi 24) + nosp : NoSp c + depth : c.depth ≤ 3 + ctl : ctlOk c = true + sp : c.all (fun i => !isa.writesSp i) = true + /-- Each function of the backend `B` meets its contract, and is safe to call. -/ structure BackendOk (B : Backend) : Prop where ntt : FnOk (fun S => Spec.MlDsa.nttContract X86_64.abi S) B.ntt @@ -39,6 +49,7 @@ structure BackendOk (B : Backend) : Prop where mulAdd : FnOk (fun S => Spec.MlDsa.mulAddContract X86_64.abi S) B.mulAdd add : FnOk (fun S => Spec.MlDsa.addContract X86_64.abi S) B.add sub : FnOk (fun S => Spec.MlDsa.subContract X86_64.abi S) B.sub + rej4 : Rej4Ok B.rej4 /-- An implementation of the polynomial arithmetic on x86-64. -/ structure ArithImpl where @@ -68,7 +79,9 @@ def ArithImpl.sse2 : ArithImpl where add := FnOk.of Arith.add_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) sub := FnOk.of Arith.sub_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) - (by decide +kernel) } + (by decide +kernel) + rej4 := ⟨Rej4.rejNTT4_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), by decide +kernel, + by decide +kernel, Code.all_of_allInstrs (by decide +kernel)⟩ } features := [] end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean index 157073d12..0036f4cdb 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean @@ -30,7 +30,9 @@ def ArithImpl.avx2 : ArithImpl where add := FnOk.of Arith.addY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) sub := FnOk.of Arith.subY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) - (by decide +kernel) } + (by decide +kernel) + rej4 := ⟨Rej4.rejNTT4Avx2_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), by decide +kernel, + by decide +kernel, Code.all_of_allInstrs (by decide +kernel)⟩ } features := ["avx", "avx2"] end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4CT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4CT.lean new file mode 100644 index 000000000..fe0aa67dc --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4CT.lean @@ -0,0 +1,177 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Top +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejNttCT +import VerifiedGarbage.Proof.MlKem.X86_64.S4CT + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4_avx2`, constant time but for the seeds + +Untrusted: everything here is checked by Lean. Two runs whose seeds (the +declared leak) and pointers agree leak the same. The code but for the loops +of the halves is proven by the taint analysis, from the pointers (the +prologue, the absorption and the first squeeze as in +`vg_mlkem_sample_ntt4_avx2`, `S4.start_ct`). Both runs read the same XOF +output, so in each half's loop they are at the same iteration with the same +coefficients sampled, and each iteration runs as in +`vg_mldsa_rej_ntt_poly` (`RejNttCT.body_ct`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Impl.MlDsa.X86_64.Sample (rnBody) +open VG.Impl.MlDsa.X86_64.Sample.Rej4 (oJ half first second zeroJ rejNTT4Avx2) +open VG.Proof.MlKem.X86_64 (sample4K relInv taintRel relStart Rel2 ofNat64_pred ofNat64_beq_zero RelCT.postDep) +open VG.Proof.MlKem.X86_64.S4 (R4 Pre aP at' pre_of pub_scr pub_aP pub_B env_rbx start_ct) +open VG.Proof.MlDsa.X86_64.Sample (nil_regs WP.all') +open VG.Proof.MlDsa.X86_64.Sample.RejNttCT (BPre BRel body_ct) +open VG.Spec.MlKem (poly4) + +theorem pub_Lt4 {σ₁ σ₂ : State} (hq : sample4K.pub σ₁ σ₂) {K : Nat} (hK : K < 4) (t : Nat) : + Lt σ₁ K t = Lt σ₂ K t := by simp only [Lt, Xb, pub_B hq hK] + +/-! ## The squeezes -/ + +theorem sqTaint0 : ∃ hc : VG.Taint.Hint X86_64.Taint.T, + (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 0) hc).isSome = true := ⟨_, by taint_decide⟩ +theorem sqTaint1 : ∃ hc : VG.Taint.Hint X86_64.Taint.T, + (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 1) hc).isSome = true := ⟨_, by taint_decide⟩ +theorem sqTaint2 : ∃ hc : VG.Taint.Hint X86_64.Taint.T, + (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 2) hc).isSome = true := ⟨_, by taint_decide⟩ + +/-- A squeeze, from the absorption, given its taint analysis. -/ +theorem sqT_ct {t n : Nat} (hn : n < 3) + (c : ∃ hc : VG.Taint.Hint X86_64.Taint.T, (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 n) hc).isSome = true) : + RelCT isa (R4 fun σ s => SqT σ t n s) (squeeze4 n) (R4 fun σ s => SqT σ t (n + 1) s) := by + obtain ⟨_, c⟩ := c + exact relInv (fun σ s hp h => WP.mono (sqT_ok (pre_of hp) hn h) fun _ h' => h'.1) + (taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) c) + +/-- A second squeeze, given its taint analysis. -/ +theorem sqM_ct {n : Nat} (hn : n < 3) + (c : ∃ hc : VG.Taint.Hint X86_64.Taint.T, (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 n) hc).isSome = true) : + RelCT isa (R4 fun σ s => M2 σ n s) (squeeze4 n) (R4 fun σ s => M2 σ (n + 1) s) := by + obtain ⟨_, c⟩ := c + exact relInv (fun σ s hp h => sqM_ok (pre_of hp) hn h) + (taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.sq.env h₂.sq.env) c) + +/-! ## The halves -/ + +/-- Two runs at iteration `t` of a half of seed `K`, from states with `X`, +`n = 168 - t` iterations from its end. -/ +def HI (X : State → State → Prop) (K h n : Nat) (s₁ s₂ : State) : Prop := + ∃ σ₁ σ₂ s₀₁ s₀₂ t, sample4K.pre σ₁ ∧ sample4K.pre σ₂ ∧ sample4K.pub σ₁ σ₂ ∧ n = 168 - t ∧ t < 168 ∧ + X σ₁ s₀₁ ∧ X σ₂ s₀₂ ∧ HAt σ₁ s₀₁ K h t s₁ ∧ HAt σ₂ s₀₂ K h t s₂ + +theorem hbpre {σ : State} (hp : Pre σ) {s₀ : State} {K h t : Nat} (hK : K < 4) (ht : t < 168) {s : State} + (hI : HAt σ s₀ K h t s) : BPre s (poly4 (aP σ) K) (Lt σ K (168 * h + t)) := + ⟨hI.rbp, hI.rdi, Lt_length_le σ K _, coeffsWr hp hK hI.env, hI.stored, + by simpa using hat_regions hp hK hI (j := 0) (by omega), hat_regions hp hK hI (by omega), + hat_regions hp hK hI (by omega)⟩ + +theorem hi_brel {X : State → State → Prop} {K h n : Nat} (hK : K < 4) (hh : h < 2) {s₁ s₂ : State} + (H : HI X K h n s₁ s₂) : BRel s₁ s₂ := by + obtain ⟨σ₁, σ₂, s₀₁, s₀₂, t, p₁, p₂, hq, _, ht, _, _, l₁, l₂⟩ := H + refine ⟨poly4 (aP σ₁) K, Lt σ₁ K (168 * h + t), hbpre (pre_of p₁) hK ht l₁, + by rw [pub_aP hq, pub_Lt4 hq hK]; exact hbpre (pre_of p₂) hK ht l₂, + by rw [l₁.rsi, l₂.rsi, at', at', pub_scr hq], by rw [l₁.rcx, l₂.rcx], fun k hk => ?_⟩ + rw [hat_byte hh l₁ (by omega), hat_byte hh l₂ (by omega)] + simp only [Xb, pub_B hq hK] + +/-- The loop of a half. -/ +theorem hloop_ct {X : State → State → Prop} {K h : Nat} (hK : K < 4) (hh : h < 2) (n : Nat) : + RelCT isa (HI X K h n) (.loop rnBody .ne) (R4 fun σ s => ∃ s₀, X σ s₀ ∧ HAt σ s₀ K h 168 s) := by + refine RelCT.loop (M := isa) (HI X K h) (fun n => ?_) n + refine RelCT.postDep (F := fun (x x' : State) => ∀ p : State × State × Nat, sample4K.pre p.1 ∧ p.2.2 < 168 ∧ + HAt p.1 p.2.1 K h p.2.2 x → + HAt p.1 p.2.1 K h (p.2.2 + 1) x' ∧ x'.zf = some (BitVec.ofNat 64 (168 - p.2.2) - 1 == 0)) + (RelCT.mono body_ct (fun x y H => hi_brel hK hh H) fun _ _ _ => trivial) (fun x y H => ?_) ?_ + · obtain ⟨σ₁, σ₂, s₀₁, s₀₂, t, p₁, p₂, _, _, ht, _, _, l₁, l₂⟩ := H + exact ⟨WP.all' (fun p hp' => hat_step (pre_of hp'.1) hK hh hp'.2.1 hp'.2.2) ⟨(σ₁, s₀₁, t), p₁, ht, l₁⟩, + WP.all' (fun p hp' => hat_step (pre_of hp'.1) hK hh hp'.2.1 hp'.2.2) ⟨(σ₂, s₀₂, t), p₂, ht, l₂⟩⟩ + · intro x y x' y' ⟨σ₁, σ₂, s₀₁, s₀₂, t, p₁, p₂, hq, hn, ht, x₁, x₂, l₁, l₂⟩ f₁ f₂ + obtain ⟨l₁', z₁⟩ := f₁ (σ₁, s₀₁, t) ⟨p₁, ht, l₁⟩ + obtain ⟨l₂', z₂⟩ := f₂ (σ₂, s₀₂, t) ⟨p₂, ht, l₂⟩ + have ez : (BitVec.ofNat 64 (168 - t) - 1 == 0) = decide (t + 1 = 168) := by + rw [ofNat64_pred (by omega) (by omega), ofNat64_beq_zero (by omega)] + exact decide_eq_decide.mpr (by omega) + rw [ez] at z₁ z₂ + refine ⟨by show x'.zf.map _ = y'.zf.map _; rw [z₁, z₂], fun hf => ?_, fun ht' => ?_⟩ + · have : t + 1 = 168 := by + have : x'.zf.map (!·) = some false := hf + rw [z₁] at this; simpa using this + rw [this] at l₁' l₂' + exact ⟨σ₁, σ₂, p₁, p₂, hq, ⟨s₀₁, x₁, l₁'⟩, ⟨s₀₂, x₂, l₂'⟩⟩ + · have : t + 1 ≠ 168 := by + have : x'.zf.map (!·) = some true := ht' + rw [z₁] at this; simpa using this + exact ⟨168 - (t + 1), by omega, σ₁, σ₂, s₀₁, s₀₂, t + 1, p₁, p₂, hq, rfl, by omega, x₁, x₂, l₁', l₂'⟩ + +/-- The setup of a half. -/ +abbrev hsetup (K : Nat) : List Instr := + [.mov .rsi (.reg .rbx), .alu .add .rsi (.imm (BitVec.ofNat 32 (oBuf + 504 * K))), + .mov .rbp (.reg .r13), .alu .add .rbp (.imm (BitVec.ofNat 32 (1024 * K))), + .mov .rdi (.mem (at_ .rbx (oJ + 8 * K))), .mov32 .rcx (.imm 168)] + +/-- A half of seed `K` from states with `X`, given the taint analysis of its setup. -/ +theorem half_ct {X : State → State → Prop} {K h : Nat} (hK : K < 4) (hh : h < 2) + (hX : ∀ σ s, sample4K.pre σ → X σ s → HPre σ K h s) {hc : VG.Taint.Hint X86_64.Taint.T} + (c : (taint.check (X86_64.Taint.ofRegs [.rbx, .r13]) (.block (hsetup K)) hc).isSome = true) : + RelCT isa (R4 X) (half K) (R4 fun σ s => ∃ s₀, X σ s₀ ∧ HAt σ s₀ K h 168 s) := by + unfold half + refine RelCT.seq (RelCT.mono (relInv (I' := fun σ s => ∃ s₀, X σ s₀ ∧ HAt σ s₀ K h 0 s) + (fun σ s hp hx => WP.mono (hsetup_ok (pre_of hp) hK (hX σ s hp hx)) fun _ h' => ⟨s, hx, h'⟩) + (taintRel [.rbx, .r13] (fun x y ⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩ r hr => by + have e₁ := (hX σ₁ x p₁ h₁).env + have e₂ := (hX σ₂ y p₂ h₂).env + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl + · exact env_rbx hq e₁ e₂ + · rw [e₁.r13, e₂.r13, pub_aP hq]) c)) (fun _ _ h => h) + (Q' := HI X K h 168) fun _ _ ⟨σ₁, σ₂, p₁, p₂, hq, ⟨s₀₁, x₁, l₁⟩, ⟨s₀₂, x₂, l₂⟩⟩ => + ⟨σ₁, σ₂, s₀₁, s₀₂, 0, p₁, p₂, hq, rfl, by decide, x₁, x₂, l₁, l₂⟩) (hloop_ct hK hh 168) + +/-- The first half of seed `K`, given the taint analysis of its blocks. -/ +theorem first_ct {K : Nat} (hK : K < 4) {h₁ h₂ : VG.Taint.Hint X86_64.Taint.T} + (c₁ : (taint.check (X86_64.Taint.ofRegs [.rbx, .r13]) (.block (hsetup K)) h₁).isSome = true) + (c₂ : (taint.check (X86_64.Taint.ofRegs [.rbx]) (.block [.store (at_ .rbx (oJ + 8 * K)) .rdi]) h₂).isSome = + true) : + RelCT isa (R4 fun σ s => P1 σ K s) (first K) (R4 fun σ s => P1 σ (K + 1) s) := + RelCT.seq (half_ct hK (by decide) (fun _ _ _ h => hpre1 hK h) c₁) + (relInv (fun σ s hp ⟨_, h₀, hA⟩ => firstEnd_ok (pre_of hp) hK h₀ hA) + (taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, ⟨_, _, a₁⟩, ⟨_, _, a₂⟩⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq a₁.env a₂.env) c₂)) + +/-- The second half of seed `K`, given the taint analysis of its setup. -/ +theorem second_ct {K : Nat} (hK : K < 4) {h₁ : VG.Taint.Hint X86_64.Taint.T} + (c₁ : (taint.check (X86_64.Taint.ofRegs [.rbx, .r13]) (.block (hsetup K)) h₁).isSome = true) : + RelCT isa (R4 fun σ s => P2 σ K s) (second K) (R4 fun σ s => P2 σ (K + 1) s) := + RelCT.seq (half_ct hK (by decide) (fun _ _ _ h => hpre2 hK h) c₁) + (relInv (fun σ s hp ⟨_, h₀, hA⟩ => secondEnd_ok (pre_of hp) hK h₀ hA) (taintRel [] nil_regs (by taint_decide))) + +/-! ## The whole function -/ + +theorem ct : ConstantTime isa r4K.pre r4K.pub rejNTT4Avx2 := by + refine relStart (Q := fun _ _ => True) (RelCT.seq start_ct (RelCT.seq (RelCT.mono (sqT_ct (t := 0) (by decide) sqTaint0) + (fun _ _ ⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩ => ⟨σ₁, σ₂, p₁, p₂, hq, sqT_of h₁, sqT_of h₂⟩) fun _ _ h => h) + (RelCT.seq (sqT_ct (by decide) sqTaint1) (RelCT.seq (sqT_ct (by decide) sqTaint2) ?_)))) + refine RelCT.seq (relInv (fun σ s hp h => zeroJ_ok (pre_of hp) h) + (taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) (by taint_decide))) ?_ + refine RelCT.seq (first_ct (K := 0) (by decide) (by taint_decide) (by taint_decide)) ?_ + refine RelCT.seq (first_ct (K := 1) (by decide) (by taint_decide) (by taint_decide)) ?_ + refine RelCT.seq (first_ct (K := 2) (by decide) (by taint_decide) (by taint_decide)) ?_ + refine RelCT.seq (first_ct (K := 3) (by decide) (by taint_decide) (by taint_decide)) ?_ + refine RelCT.seq (RelCT.mono (sqM_ct (by decide) sqTaint0) + (fun _ _ ⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩ => ⟨σ₁, σ₂, p₁, p₂, hq, m2_of h₁, m2_of h₂⟩) fun _ _ h => h) + (RelCT.seq (sqM_ct (by decide) sqTaint1) (RelCT.seq (sqM_ct (by decide) sqTaint2) ?_)) + refine RelCT.seq (relInv (fun σ s _ h => vz_ok h) (taintRel [] nil_regs (by taint_decide))) ?_ + refine RelCT.seq (second_ct (K := 0) (by decide) (by taint_decide)) ?_ + refine RelCT.seq (second_ct (K := 1) (by decide) (by taint_decide)) ?_ + refine RelCT.seq (second_ct (K := 2) (by decide) (by taint_decide)) ?_ + refine RelCT.seq (second_ct (K := 3) (by decide) (by taint_decide)) ?_ + exact taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) (by taint_decide) + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Parse.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Parse.lean new file mode 100644 index 000000000..da11e4f01 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Parse.lean @@ -0,0 +1,215 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Sq +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejNtt + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4_avx2`, sampling + +Untrusted: everything here is checked by Lean. Each half of a seed's +sampling (`half`) runs 168 iterations of `vg_mldsa_rej_ntt_poly`'s loop +(`rnBody_ok`) on the 504 bytes of the seed's buffer, which hold bytes +`504 h` to `504 h + 503` of its output `Xb` (`G` of the seed, as +`vg_mldsa_rej_ntt_poly` squeezes it: `Xb_getD`): from `j` coefficients +sampled, it samples those of the first `504 (h + 1)` bytes (`half_ok`), and +writes only the seed's polynomial. +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Impl.MlDsa.X86_64.Sample (rnBody) +open VG.Impl.MlDsa.X86_64.Sample.Rej4 (oJ half) +open VG.Proof.MlKem.X86_64 (Keep ea_at wp_countdown add_ofNat_zero pR WP.keep ofNat64_pred) +open VG.Proof.MlKem.X86_64.S4 +open VG.Proof.MlDsa.Sample (rnFold rnStep rnFold_snoc rnFold_length_le Stored stored_nil stored_frame G_eq G_length) +open VG.Proof.MlDsa.X86_64.Sample (rnBody_ok CoeffsWr) +open VG.Spec.MlKem (poly4 seed4) +open VG.Spec.MlDsa (Zq) +open VG.Proof.MlKem (xofByte xof_squeezeFrom_getElem) + +/-- ML-KEM's `S4.Env`, which this function shares. -/ +abbrev EnvK := VG.Proof.MlKem.X86_64.S4.Env + +/-- The 1008 bytes of output of seed `k`, which `vg_mldsa_rej_ntt_poly` samples from. -/ +abbrev Xb (σ : State) (k : Nat) : List Byte := Spec.MlDsa.G (B σ k) 1008 + +theorem Xb_getD (σ : State) (k : Nat) {p : Nat} (hp : p < 1008) : (Xb σ k).getD p 0 = xofByte (B σ k) p := by + have e := xof_squeezeFrom_getElem (B σ k) (pos := 0) (d := 1008) hp + rw [Nat.zero_add] at e + rw [List.getD_eq_getElem?_getD, List.getElem?_eq_getElem (by rw [Xb, G_length]; exact hp), Option.getD_some, ← e] + simp only [Xb, G_eq] + +/-- The coefficients of seed `k` after `n` iterations. -/ +abbrev Lt (σ : State) (k n : Nat) : List Zq := rnFold [] ((Xb σ k).take (3 * n)) + +theorem Lt_length_le (σ : State) (k n : Nat) : (Lt σ k n).length ≤ 256 := rnFold_length_le (by simp) _ + +theorem sx_ofNat {n : Nat} (h : n < 2 ^ 31) : BitVec.signExtend 64 (BitVec.ofNat 32 n) = BitVec.ofNat 64 n := by + have hm : (BitVec.ofNat 32 n).msb = false := by + rw [BitVec.msb_eq_decide]; simp only [BitVec.toNat_ofNat, decide_eq_false_iff_not]; omega + rw [BitVec.signExtend_eq_setWidth_of_msb_false hm] + apply BitVec.eq_of_toNat_eq + rw [BitVec.toNat_setWidth, BitVec.toNat_ofNat, BitVec.toNat_ofNat] + omega + +/-- A byte the frame's regions are apart from is unchanged. -/ +theorem frame_byte {rs : List Region} {m m' : Mem} (hf : Frame rs m m') {R : Region} (hd : ∀ r ∈ rs, R.Disjoint r) + {x : Addr} (hx : R.Contains x 1) : m' x = m x := + hf x fun r hr hc => hd r hr x hx hc + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +/-- Polynomial `K` lies in `a`. -/ +theorem sub_poly {K : Nat} (hK : K < 4) : Region.Sub (pR (poly4 (aP σ) K)) (aR σ) := Offset.sub_base _ (by omega) + +/-- A part of the scratch space is apart from polynomial `K`. -/ +theorem scr_poly {K : Nat} (hK : K < 4) {a n : Nat} (h : a + n ≤ 8192) : + ∀ r ∈ [pR (poly4 (aP σ) K)], Region.Disjoint ⟨at' σ a, n⟩ r := by + intro r hr + rw [List.mem_singleton.mp hr] + exact Region.Disjoint.sub_right (Region.Disjoint.sub_left hp.a_scr.symm (sub_scr h)) (sub_poly hK) + +omit hp in +/-- Two polynomials are apart. -/ +theorem poly_poly {K k : Nat} (hK : K < 4) (hk : k < 4) (hne : k ≠ K) : + ∀ r ∈ [pR (poly4 (aP σ) K)], (VG.Proof.MlDsa.Sample.polyR (poly4 (aP σ) k)).Disjoint r := by + intro r hr + simp only [List.mem_singleton] at hr; subst hr + exact Offset.disjoint (aP σ) (d := 1024 * k) (n := 1024) (e := 1024 * K) (k := 1024) (by omega) (by omega) + (by omega) + +/-- Writes to polynomial `K` keep `Env`. -/ +theorem env_poly {K : Nat} (hK : K < 4) {s s' : State} (he : EnvK σ s) + (hf : Frame [pR (poly4 (aP σ) K)] s.mem s'.mem) (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) + (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15], s'.gpr r = s.gpr r) : EnvK σ s' := by + have hsub := sub_poly (σ := σ) hK + refine ⟨hrd.trans he.rd, hwr.trans he.wr, by rw [hg .rbx (by simp), he.rbx], by rw [hg .r12 (by simp), he.r12], + by rw [hg .r13 (by simp), he.r13], by rw [hg .rsp (by simp), he.rsp], by rw [hg .r15 (by simp), he.r15], + fun i hi => ?_, he.frame.trans (hf.sub fun r hr => ?_)⟩ + · rw [hf.readW (Region.contains_self _ _) (scr_poly hp hK (by simp only [oSave]; omega)) (by decide)] + exact he.saved i hi + · simp only [List.mem_singleton] at hr; subst hr; exact ⟨aR σ, by simp, hsub⟩ + +/-- Each coefficient of polynomial `K` is writable. -/ +theorem coeffsWr {K : Nat} (hK : K < 4) {s : State} (he : EnvK σ s) : CoeffsWr s.wr (poly4 (aP σ) K) := by + intro j hj + rw [he.wr, hp.wr] + refine ⟨aR σ, by simp, ?_⟩ + rw [VG.Proof.MlDsa.Sample.coeffAddr, poly4, Offset.add_add] + exact Offset.contains_base _ (by omega) (by omega) + +end + +/-! ## A half -/ + +/-- At iteration `t` of half `h` of seed `K`, from `s₀`. -/ +structure HAt (σ s₀ : State) (K h t : Nat) (s : State) : Prop where + env : EnvK σ s + rsi : s.gpr .rsi = at' σ (oBuf + 504 * K) + BitVec.ofNat 64 (3 * t) + rdi : s.gpr .rdi = BitVec.ofNat 64 (Lt σ K (168 * h + t)).length + rcx : s.gpr .rcx = BitVec.ofNat 64 (168 - t) + rbp : s.gpr .rbp = poly4 (aP σ) K + out : ∀ p < 504, s.mem (at' σ (oBuf + 504 * K + p)) = xofByte (B σ K) (504 * h + p) + stored : Stored s.mem (poly4 (aP σ) K) (Lt σ K (168 * h + t)) + fr : Frame [pR (poly4 (aP σ) K)] s₀.mem s.mem + kp : Keep [.rax, .rdx, .r8, .rdi, .rsi, .rcx, .rbp] s₀ s + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +theorem hat_byte {s₀ : State} {K h t : Nat} (hh : h < 2) {s : State} (hI : HAt σ s₀ K h t s) {j : Nat} + (hj : 3 * t + j < 504) : + s.mem (s.gpr .rsi + BitVec.ofNat 64 j) = (Xb σ K).getD (3 * (168 * h + t) + j) 0 := by + have e := hI.out (3 * t + j) hj + rw [at'] at e + rw [hI.rsi, at', Offset.add_add, Offset.add_add, e, Xb_getD σ K (by omega)] + congr 1; omega + +theorem hat_regions {s₀ : State} {K h t : Nat} (hK : K < 4) {s : State} (hI : HAt σ s₀ K h t s) {j : Nat} + (hj : 3 * t + j < 504) : + InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 j) 1 := by + rw [hI.rsi, at', Offset.add_add, Offset.add_add] + exact in_scr' hp hI.env.rd hI.env.wr (by simp only [oBuf]; omega) + +/-- An iteration. -/ +theorem hat_step {s₀ : State} {K h t : Nat} (hK : K < 4) (hh : h < 2) (ht : t < 168) {s : State} + (hI : HAt σ s₀ K h t s) : + WP isa rnBody s fun s' => HAt σ s₀ K h (t + 1) s' ∧ s'.zf = some (BitVec.ofNat 64 (168 - t) - 1 == 0) := by + refine WP.mono (rnBody_ok s (aP := poly4 (aP σ) K) hI.rbp hI.rdi (Lt_length_le σ K _) (coeffsWr hp hK hI.env) + hI.stored (by simpa using hat_regions hp hK hI (j := 0) (by omega)) (hat_regions hp hK hI (by omega)) + (hat_regions hp hK hI (by omega))) fun s' ⟨hdi, hst, hf, hsi, hcx, hz, hk⟩ => ?_ + have e0 := hat_byte hh hI (j := 0) (by omega) + rw [add_ofNat_zero, Nat.add_zero] at e0 + have ht3 : Lt σ K (168 * h + (t + 1)) = rnStep (Lt σ K (168 * h + t)) ((Xb σ K).getD (3 * (168 * h + t)) 0) + ((Xb σ K).getD (3 * (168 * h + t) + 1) 0) ((Xb σ K).getD (3 * (168 * h + t) + 2) 0) := by + simp only [Lt] + rw [show 3 * (168 * h + (t + 1)) = 3 * (168 * h + t) + 3 by omega, + VG.Proof.MlDsa.X86_64.Sample.RejNtt.take_add_three _ (by rw [Xb, G_length]; omega), + rnFold_snoc _ (by rw [List.length_take, Xb, G_length]; omega)] + rw [e0, hat_byte hh hI (j := 1) (by omega), hat_byte hh hI (j := 2) (by omega), ← ht3] at hdi hst + have hk' : Keep [.rax, .rdx, .r8, .rdi, .rsi, .rcx] s s' := hk + refine ⟨⟨env_poly hp hK hI.env hf hk'.2.1 hk'.2.2 fun r hr => hk'.gpr (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr; rcases hr with rfl | rfl | rfl | rfl | rfl <;> decide), + by rw [hsi, hI.rsi, Offset.add_add, show 3 * t + 3 = 3 * (t + 1) by omega], hdi, + by rw [hcx, hI.rcx, ofNat64_pred (by omega) (by omega)]; rfl, + by rw [hk'.gpr (by decide), hI.rbp], + fun p hp' => by + rw [frame_byte hf (scr_poly hp hK (a := oBuf + 504 * K + p) (n := 1) (by simp only [oBuf]; omega)) + (Region.contains_self _ _)] + exact hI.out p hp', + hst, hI.fr.trans hf, (hI.kp.trans hk').mono (by simp)⟩, by rw [hz, hI.rcx]⟩ + +/-- Before a half: what holds of seed `K`. -/ +structure HPre (σ : State) (K h : Nat) (s : State) : Prop where + env : EnvK σ s + out : ∀ p < 504, s.mem (at' σ (oBuf + 504 * K + p)) = xofByte (B σ K) (504 * h + p) + j : s.mem.readW (at' σ (oJ + 8 * K)) 64 = BitVec.ofNat 64 (Lt σ K (168 * h)).length + stored : Stored s.mem (poly4 (aP σ) K) (Lt σ K (168 * h)) + +omit hp in +theorem sxB {K : Nat} (hK : K < 4) : BitVec.signExtend 64 (BitVec.ofNat 32 (oBuf + 504 * K)) = + BitVec.ofNat 64 (oBuf + 504 * K) := sx_ofNat (by simp only [oBuf]; omega) + +omit hp in +theorem sxP {K : Nat} (hK : K < 4) : BitVec.signExtend 64 (BitVec.ofNat 32 (1024 * K)) = + BitVec.ofNat 64 (1024 * K) := sx_ofNat (by omega) + +/-- The setup of a half: at iteration 0, from `j` kept. -/ +theorem hsetup_ok {K h : Nat} (hK : K < 4) {s : State} (hI : HPre σ K h s) : + WP isa (.block [.mov .rsi (.reg .rbx), .alu .add .rsi (.imm (BitVec.ofNat 32 (oBuf + 504 * K))), + .mov .rbp (.reg .r13), .alu .add .rbp (.imm (BitVec.ofNat 32 (1024 * K))), + .mov .rdi (.mem (at_ .rbx (oJ + 8 * K))), .mov32 .rcx (.imm 168)]) s (HAt σ s K h 0) := by + have hin : InRegions (s.rd ++ s.wr) (scr σ + BitVec.ofNat 64 (oJ + 8 * K)) 8 := + in_scr' hp hI.env.rd hI.env.wr (by simp only [oJ]; omega) + refine WP.mono (WP.keep [.rsi, .rbp, .rdi, .rcx] (Q := fun s' => s'.mem = s.mem ∧ + s'.gpr .rsi = at' σ (oBuf + 504 * K) ∧ s'.gpr .rbp = poly4 (aP σ) K ∧ + s'.gpr .rdi = BitVec.ofNat 64 (Lt σ K (168 * h)).length ∧ s'.gpr .rcx = BitVec.ofNat 64 168) + (by + xrun [hI.env.rbx, hI.env.r13, sxB hK, sxP hK, hin, hI.j] + rfl) + rfl) fun s₁ ⟨⟨hm, hsi, hbp, hdi, hcx⟩, k₁⟩ => ?_ + exact ⟨hI.env.keep hm k₁ (by decide), by rw [hsi, add_ofNat_zero], + by rw [hdi, Nat.add_zero], by rw [hcx], hbp, by rw [hm]; exact hI.out, by rw [hm, Nat.add_zero]; exact hI.stored, + by rw [hm]; exact Frame.refl _ _, k₁.mono (by simp)⟩ + +/-- The loop of a half, from iteration 0. -/ +theorem hloop_ok {s₀ : State} {K h : Nat} (hK : K < 4) (hh : h < 2) {s : State} (hI : HAt σ s₀ K h 0 s) : + WP isa (.loop rnBody .ne) s (HAt σ s₀ K h 168) := by + refine wp_countdown (N := 168) (by decide) (by decide) (fun t u => HAt σ s₀ K h t u) + (fun t ht u hu _ => WP.mono (hat_step hp hK hh ht hu) fun u' ⟨hu', hz⟩ => ⟨hu', ?_, by rw [hz, hu.rcx]⟩) + (fun _ h => h) hI hI.rcx + rw [hu'.rcx, hu.rcx, ofNat64_pred (by omega) (by omega)]; rfl + +/-- A half of seed `K`: the coefficients of the first `504 (h + 1)` bytes, +writing only polynomial `K`. -/ +theorem half_ok {K h : Nat} (hK : K < 4) (hh : h < 2) {s : State} (hI : HPre σ K h s) : + WP isa (half K) s (HAt σ s K h 168) := + WP.seq (WP.mono (hsetup_ok hp hK hI) fun _ h₁ => hloop_ok hp hK hh h₁) + +end + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Scalar.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Scalar.lean new file mode 100644 index 000000000..acc7780d0 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Scalar.lean @@ -0,0 +1,264 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4CT +import VerifiedGarbage.Proof.MlDsa.Arith.Mem + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4` + +Untrusted: everything here is checked by Lean. The baseline implementation +calls `vg_mldsa_rej_ntt_poly` on each seed, between the prologue and the +epilogue of the one for AVX2 (`Rej4Top.lean`): after the call on seed `K`, +polynomial `K` is the seed's `RejNTTPoly` if it has 256 coefficients, and +`r14` records whether the first `K + 1` do (`PC`), as in +`vg_mlkem_sample_ntt4` (`MlKem/X86_64/S4Scalar.lean`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Impl.MlDsa.X86_64.Sample.Rej4 (rejNTT4) +open VG.Proof.MlKem.X86_64 (Keep sample4K relInv taintRel relStart Rel2 pR ret_disj24 stk_disj24' + ce_bytesAt24 ce_gpr' WP.keep nosp_of) +open VG.Proof.MlKem.X86_64.S4 (R4 Pre aP at' sd scr aR scrR stkR sdR pre_of pub_scr pub_aP pub_B pub_sd pub_sp + env_rbx pro_ok I0 sub_scr sub_poly seed_bytes cov scr6144_lt sx34 sx1024 sx6144 c_sub c_disj and14_ok epi_eq + cRd cWr in_scr') +open VG.Proof.MlDsa.X86_64.Sample (rnK nil_regs rejNTT_correct rejNTT_ct) +open VG.Proof.MlDsa.Sample (rnFold toPoly) +open VG.Proof.MlDsa.Arith (polyIs_frame) +open VG.Spec.MlKem (poly4) +open VG.Spec.MlDsa (G PolyIs) + +theorem rn_nosp : NoSp Impl.MlDsa.X86_64.Sample.rejNTT := nosp_of (by decide +kernel) + +theorem rn_depth : Impl.MlDsa.X86_64.Sample.rejNTT.depth = 2 := by decide +kernel + +/-- Before the call on seed `K`. -/ +structure PC (σ : State) (K : Nat) (s : State) : Prop where + env : EnvK σ s + r14 : s.gpr .r14 = BitVec.ofNat 64 (okN σ K) + polys : ∀ k < K, (Lt σ k 336).length = 256 → PolyIs s.mem (poly4 (aP σ) k) (toPoly (Lt σ k 336)) + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +/-- `PC` after code that writes no memory and keeps its registers. -/ +theorem PC.keep {K : Nat} {s s' : State} (h : PC σ K s) (hm : s'.mem = s.mem) {rs : List Reg} + (hk : Keep rs s s') (hrs : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15, .r14], r ∉ rs) : PC σ K s' := + ⟨h.env.keep hm hk fun r hr => hrs r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr ⊢; rcases hr with h | h | h | h | h <;> simp [h]), + by rw [hk.gpr (hrs .r14 (by simp)), h.r14], by rw [hm]; exact h.polys⟩ + +/-- `PC` after the call. -/ +theorem PC.call {K : Nat} (hK : K < 4) {s s' : State} (h : PC σ K s) (hrd : s'.rd = s.rd) + (hwr : s'.wr = s.wr) (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15, .r14], s'.gpr r = s.gpr r) + (hf : Frame (cWr σ K ++ [stkR σ]) s.mem s'.mem) : PC σ K s' := by + refine ⟨⟨hrd.trans h.env.rd, hwr.trans h.env.wr, by rw [hg .rbx (by simp), h.env.rbx], + by rw [hg .r12 (by simp), h.env.r12], by rw [hg .r13 (by simp), h.env.r13], by rw [hg .rsp (by simp), h.env.rsp], + by rw [hg .r15 (by simp), h.env.r15], fun i hi => ?_, h.env.frame.trans (hf.sub (c_sub (σ := σ) hK))⟩, + by rw [hg .r14 (by simp), h.r14], fun k hk e => ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (c_disj hp hK (by simp only [oSave, oScalar]; omega)) (by decide)] + exact h.env.saved i hi + · refine polyIs_frame hf (fun r hr => ?_) (h.polys k hk e) + simp only [List.mem_append, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with (rfl | rfl) | rfl + · have hd := Offset.disjoint (aP σ) (d := 1024 * k) (n := 1024) (e := 1024 * K) (k := 1024) (by omega) (by omega) + (by omega) + simpa [poly4] using hd + · exact (hp.a_scr.sub_left (sub_poly (by omega))).sub_right (sub_scr (by simp only [oScalar]; omega)) + · exact (hp.stk_a.sub_right (sub_poly (by omega))).symm + +/-- The arguments of the call on seed `K`. -/ +structure ArgI (σ : State) (K : Nat) (s : State) : Prop where + pinv : PC σ K s + rdi : s.gpr .rdi = sd σ + BitVec.ofNat 64 (34 * K) + rsi : s.gpr .rsi = poly4 (aP σ) K + rdx : s.gpr .rdx = at' σ oScalar + +omit hp in +theorem argsK_ok {K : Nat} (hK : K < 4) {s : State} (h : PC σ K s) : + WP isa (.block [.mov .rdi (.reg .r12), .alu .add .rdi (.imm (BitVec.ofNat 32 (34 * K))), + .mov .rsi (.reg .r13), .alu .add .rsi (.imm (BitVec.ofNat 32 (1024 * K))), .mov .rdx (.reg .rbx), + .alu .add .rdx (.imm (BitVec.ofNat 32 oScalar))]) s (ArgI σ K) := + WP.mono (WP.keep [.rdi, .rsi, .rdx] (Q := fun s' => s'.mem = s.mem ∧ + s'.gpr .rdi = sd σ + BitVec.ofNat 64 (34 * K) ∧ s'.gpr .rsi = poly4 (aP σ) K ∧ s'.gpr .rdx = at' σ oScalar) + (by xrun [h.env.r12, h.env.r13, h.env.rbx, sx34 hK, sx1024 hK, sx6144]; exact ⟨rfl, rfl⟩) rfl) + fun _ ⟨⟨hm₂, hdi, hsi, hdx⟩, k₂⟩ => ⟨h.keep hm₂ k₂ (by decide), hdi, hsi, hdx⟩ + +theorem argK_kS {K : Nat} (hK : K < 4) {s : State} (h : ArgI σ K s) : + (below (s.gpr .rsp) 24).Disjoint ⟨sd σ + BitVec.ofNat 64 (34 * K), 34⟩ := by + rw [h.pinv.env.rsp]; exact hp.stk_sd.sub_right (Offset.sub_base (sd σ) (d := 34 * K) (n := 34) (by omega)) + +theorem argK_pre {K : Nat} (hK : K < 4) {s : State} (h : ArgI σ K s) : + rnK.pre (s.callEntry.withRegions (cRd σ K) (cWr σ K)) := by + have hsp : s.gpr .rsp = σ.gpr .rsp := h.pinv.env.rsp + have kS := argK_kS hp hK h + have kA : (below (s.gpr .rsp) 24).Disjoint (pR (poly4 (aP σ) K)) := by + rw [hsp]; exact hp.stk_a.sub_right (sub_poly (σ := σ) hK) + have kZ : (below (s.gpr .rsp) 24).Disjoint ⟨at' σ oScalar, 2048⟩ := by + rw [hsp]; exact hp.stk_scr.sub_right (sub_scr (σ := σ) (a := oScalar) (n := 2048) (by simp only [oScalar]; omega)) + simp only [rnK, State.withRegions_gpr, State.withRegions_rd, State.withRegions_wr, + ce_gpr' s (by decide : Reg.rdi ≠ .rsp), ce_gpr' s (by decide : Reg.rsi ≠ .rsp), + ce_gpr' s (by decide : Reg.rdx ≠ .rsp), h.rdi, h.rsi, h.rdx] + exact ⟨trivial, trivial, + (hp.sd_a.sub_left (Offset.sub_base _ (by omega))).sub_right (sub_poly hK), + (hp.sd_scr.sub_left (Offset.sub_base _ (by omega))).sub_right (sub_scr (by simp only [oScalar]; omega)), + (hp.a_scr.sub_left (sub_poly hK)).sub_right (sub_scr (by simp only [oScalar]; omega)), + ret_disj24 s kS, ret_disj24 s kA, ret_disj24 s kZ, stk_disj24' s kS, stk_disj24' s kA, stk_disj24' s kZ, + scr6144_lt hp⟩ + +/-- After the call on seed `K`. -/ +structure CallI (σ : State) (K : Nat) (s : State) : Prop where + pinv : PC σ K s + rax : (s.gpr .rax).setWidth 32 = if (Lt σ K 336).length = 256 then 1 else 0 + poly : (Lt σ K 336).length = 256 → PolyIs s.mem (poly4 (aP σ) K) (toPoly (Lt σ K 336)) + +theorem callK_ok {K : Nat} (hK : K < 4) {s : State} (h : ArgI σ K s) : + WP isa (.call "vg_mldsa_rej_ntt_poly" Impl.MlDsa.X86_64.Sample.rejNTT) s (CallI σ K) := by + have hcv := cov hp h.pinv.env hK + refine WP.call rejNTT_correct rn_nosp (by rw [rn_depth]; decide) (argK_pre hp hK h) hcv.1 hcv.2 + fun s₃ hrd hwr hcs hf _ ⟨s₃', hm₃, hg₃, hpost⟩ => ?_ + rw [rn_depth, h.pinv.env.rsp] at hf + have h₃ : PC σ K s₃ := h.pinv.call hp hK hrd hwr (fun r hr => hcs r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide)) hf + simp only [rnK, State.withRegions_gpr, State.withRegions_mem, ce_gpr' s (by decide : Reg.rdi ≠ .rsp), + ce_gpr' s (by decide : Reg.rsi ≠ .rsp), h.rdi, h.rsi, hm₃, ce_bytesAt24 s (n := 34) (by decide) (argK_kS hp hK h), + seed_bytes hp hK h.pinv.env.frame] at hpost + rw [hg₃ .rax (by decide)] at hpost + simp only [← Lt_336] at hpost + exact ⟨h₃, hpost.1, hpost.2⟩ + +omit hp in +theorem okN_succ' (K : Nat) : BitVec.setWidth 64 ((BitVec.ofNat 64 (okN σ K)).setWidth 32 &&& + (if (Lt σ K 336).length = 256 then 1 else 0)) = BitVec.ofNat 64 (okN σ (K + 1)) := by + simp only [okN, List.range_succ, List.all_append, List.all_cons, List.all_nil, Bool.and_true] + by_cases e : (Lt σ K 336).length = 256 + · rw [ite_eq_left e, show ((Lt σ K 336).length == 256) = true by simpa using e] + cases (List.range K).all fun k => (Lt σ k 336).length == 256 <;> rfl + · rw [ite_eq_right e, show ((Lt σ K 336).length == 256) = false by simpa using e] + cases (List.range K).all fun k => (Lt σ k 336).length == 256 <;> rfl + +omit hp in +theorem andK_ok {K : Nat} {s : State} (h : CallI σ K s) : + WP isa (.block [.alu32 .and .r14 (.reg .rax)]) s (PC σ (K + 1)) := by + refine WP.mono (and14_ok s) fun s₄ ⟨⟨h14, hm₄⟩, k₄⟩ => ?_ + refine ⟨h.pinv.env.keep hm₄ k₄ (by decide), by rw [h14, h.pinv.r14, h.rax, okN_succ'], fun k hk e => ?_⟩ + rw [hm₄] + by_cases ek : k = K + · subst ek; exact h.poly e + · exact h.pinv.polys k (by omega) e + +theorem callK_ok' {K : Nat} (hK : K < 4) {s : State} (h : PC σ K s) : WP isa (Impl.MlDsa.X86_64.Sample.Rej4.callK K) s (PC σ (K + 1)) := + WP.seq (WP.mono (argsK_ok hK h) fun _ h₂ => WP.seq (WP.mono (callK_ok hp hK h₂) fun _ h₃ => andK_ok h₃)) + +/-- The return value, and the callee-saved registers restored. -/ +theorem endS_ok {s : State} (h : PC σ 4 s) : + WP isa (.block epi) s fun s' => r4K.post σ s' ∧ gprPreserved σ s' := by + have hin : ∀ i < 5, InRegions (s.rd ++ s.wr) (scr σ + BitVec.ofNat 64 (oSave + 8 * i)) 8 := fun i hi => + in_scr' hp h.env.rd h.env.wr (by simp only [oSave]; omega) + rw [epi_eq] + refine WP.mono (WP.keep [.rax, .r14, .r13, .r12, .rbp, .rbx] (Q := fun s' => s'.mem = s.mem ∧ + (s'.gpr .rax).setWidth 32 = (s.gpr .r14).setWidth 32 ∧ + s'.gpr .r14 = s.mem.readW (at' σ (oSave + 8 * 4)) 64 ∧ s'.gpr .r13 = s.mem.readW (at' σ (oSave + 8 * 3)) 64 ∧ + s'.gpr .r12 = s.mem.readW (at' σ (oSave + 8 * 2)) 64 ∧ s'.gpr .rbp = s.mem.readW (at' σ (oSave + 8 * 1)) 64 ∧ + s'.gpr .rbx = s.mem.readW (at' σ (oSave + 8 * 0)) 64) + (by + have h0 := hin 0 (by decide); have h1 := hin 1 (by decide); have h2 := hin 2 (by decide) + have h3 := hin 3 (by decide); have h4 := hin 4 (by decide) + simp only [oSave, Nat.reduceMul, Nat.reduceAdd] at h0 h1 h2 h3 h4 + xrun [h0, h1, h2, h3, h4, h.env.rbx] + exact ⟨rfl, rfl, rfl, rfl, rfl⟩) + (by decide)) fun s' ⟨⟨hm, hax, h14, h13, h12, hbp, hbx⟩, k⟩ => ?_ + refine ⟨⟨?_, fun k hk e => ?_⟩, fun r hr => ?_, ?_⟩ + · rw [hax, h.r14, okN] + simp only [Lt_336] + split <;> rfl + · rw [hm, ← Lt_336] + rw [← Lt_336] at e + exact h.polys k hk e + · simp only [calleeSaved, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl | rfl + · rw [hbx]; exact h.env.saved 0 (by decide) + · rw [hbp]; exact h.env.saved 1 (by decide) + · rw [k.gpr (by decide)]; exact h.env.rsp + · rw [h12]; exact h.env.saved 2 (by decide) + · rw [h13]; exact h.env.saved 3 (by decide) + · rw [h14]; exact h.env.saved 4 (by decide) + · rw [k.gpr (by decide)]; exact h.env.r15 + · rw [hm] + exact h.env.frame.readW (Region.contains_self _ _) (by + simpa using ⟨hp.ret_a, hp.ret_scr, Offset.base_disjoint_below (σ.gpr .rsp) (n := 24) (k := 8) (by omega)⟩) + (by decide) + +theorem scalar_body_ok {s : State} (h : I0 σ s) : + WP isa (.seq (Impl.MlDsa.X86_64.Sample.Rej4.callK 0) (.seq (Impl.MlDsa.X86_64.Sample.Rej4.callK 1) + (.seq (Impl.MlDsa.X86_64.Sample.Rej4.callK 2) (.seq (Impl.MlDsa.X86_64.Sample.Rej4.callK 3) (.block epi))))) + s fun s' => + r4K.post σ s' ∧ gprPreserved σ s' := by + have p₀ : PC σ 0 s := ⟨h.env, by rw [h.r14]; rfl, fun _ h _ => absurd h (by omega)⟩ + exact WP.seq (WP.mono (callK_ok' hp (by decide) p₀) fun _ p₁ => WP.seq (WP.mono (callK_ok' hp (by decide) p₁) + fun _ p₂ => WP.seq (WP.mono (callK_ok' hp (by decide) p₂) fun _ p₃ => + WP.seq (WP.mono (callK_ok' hp (by decide) p₃) fun _ p₄ => endS_ok hp p₄)))) + +end + +theorem correct_scalar (σ : State) (hs : r4K.pre σ) : + ∃ t s', Exec isa rejNTT4 σ t s' ∧ abiPreserved σ s' ∧ r4K.post σ s' := by + have hp := pre_of hs + obtain ⟨t, s', he, hF⟩ := WP.seq (WP.mono (pro_ok hp) fun _ h => scalar_body_ok hp h) + exact ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he hF.2, hF.1⟩ + +/-! ## Constant time -/ + +/-- The call on seed `K`. -/ +theorem call_ct {K : Nat} (hK : K < 4) : + RelCT isa (R4 fun σ s => ArgI σ K s) (.call "vg_mldsa_rej_ntt_poly" Impl.MlDsa.X86_64.Sample.rejNTT) + (R4 fun σ s => CallI σ K s) := + relInv (fun σ s hp h => callK_ok (pre_of hp) hK h) (RelCT.callEx rejNTT_correct rejNTT_ct + fun s₁ s₂ ⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩ => by + have hsp : s₁.gpr .rsp = s₂.gpr .rsp := by rw [h₁.pinv.env.rsp, h₂.pinv.env.rsp, pub_sp hq] + refine ⟨_, _, _, _, argK_pre (pre_of p₁) hK h₁, argK_pre (pre_of p₂) hK h₂, ?_, + (cov (pre_of p₁) h₁.pinv.env hK).1, (cov (pre_of p₁) h₁.pinv.env hK).2, + (cov (pre_of p₂) h₂.pinv.env hK).1, (cov (pre_of p₂) h₂.pinv.env hK).2, hsp⟩ + simp only [rnK, State.withRegions_gpr, State.withRegions_mem, State.callEntry_rsp, + ce_gpr' _ (by decide : Reg.rdi ≠ .rsp), ce_gpr' _ (by decide : Reg.rsi ≠ .rsp), + ce_gpr' _ (by decide : Reg.rdx ≠ .rsp), h₁.rdi, h₂.rdi, h₁.rsi, h₂.rsi, h₁.rdx, h₂.rdx] + rw [ce_bytesAt24 s₁ (n := 34) (by decide) (argK_kS (pre_of p₁) hK h₁), + ce_bytesAt24 s₂ (n := 34) (by decide) (argK_kS (pre_of p₂) hK h₂), + seed_bytes (pre_of p₁) hK h₁.pinv.env.frame, seed_bytes (pre_of p₂) hK h₂.pinv.env.frame, pub_B hq hK] + simp only [pub_sd hq, pub_aP hq, at', pub_scr hq, hsp, and_self]) + +/-- The call on seed `K`, given the taint analysis of its arguments. -/ +theorem callK_ct {K : Nat} (hK : K < 4) {hc : VG.Taint.Hint X86_64.Taint.T} + (c : (taint.check (X86_64.Taint.ofRegs [.r12, .r13, .rbx]) + (.block [.mov .rdi (.reg .r12), .alu .add .rdi (.imm (BitVec.ofNat 32 (34 * K))), + .mov .rsi (.reg .r13), .alu .add .rsi (.imm (BitVec.ofNat 32 (1024 * K))), .mov .rdx (.reg .rbx), + .alu .add .rdx (.imm (BitVec.ofNat 32 oScalar))]) hc).isSome = true) : + RelCT isa (R4 fun σ s => PC σ K s) (Impl.MlDsa.X86_64.Sample.Rej4.callK K) (R4 fun σ s => PC σ (K + 1) s) := + RelCT.seq (relInv (fun σ s _ h => argsK_ok hK h) (taintRel [.r12, .r13, .rbx] + (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl + · rw [h₁.env.r12, h₂.env.r12, pub_sd hq] + · rw [h₁.env.r13, h₂.env.r13, pub_aP hq] + · exact env_rbx hq h₁.env h₂.env) c)) + (RelCT.seq (call_ct hK) (relInv (fun σ s _ h => andK_ok h) (taintRel [] nil_regs (by taint_decide)))) + +theorem ct_scalar : ConstantTime isa r4K.pre r4K.pub rejNTT4 := by + refine relStart (Q := fun _ _ => True) (RelCT.seq (RelCT.mono (relInv (I' := fun σ s => PC σ 0 s) + (fun σ s hp h => by + subst h + exact WP.mono (pro_ok (pre_of hp)) fun _ h => ⟨h.env, by rw [h.r14]; rfl, fun _ h _ => absurd h (by omega)⟩) + (taintRel [.rdi, .rsi, .rdx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + subst h₁ h₂ + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl + exacts [hq.1, hq.2.1, hq.2.2.1]) (by taint_decide))) (fun _ _ h => h) fun _ _ h => h) ?_) + refine RelCT.seq (callK_ct (K := 0) (by decide) (by taint_decide)) (RelCT.seq (callK_ct (K := 1) (by decide) + (by taint_decide)) (RelCT.seq (callK_ct (K := 2) (by decide) (by taint_decide)) + (RelCT.seq (callK_ct (K := 3) (by decide) (by taint_decide)) ?_))) + exact taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) (by taint_decide) + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Sq.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Sq.lean new file mode 100644 index 000000000..e5abcc586 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Sq.lean @@ -0,0 +1,156 @@ +import VerifiedGarbage.Proof.MlKem.X86_64.S4Squeeze +import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejNtt4 + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4_avx2`, squeezing + +Untrusted: everything here is checked by Lean. `vg_mldsa_rej_ntt_poly4_avx2` +absorbs the seeds and squeezes three blocks of each as +`vg_mlkem_sample_ntt4_avx2` does (`Proof/MlKem/X86_64/S4*.lean`, whose +precondition, layout and invariants it shares), then three more to the same +buffers. `SqT σ t n` is `SqInv σ n` (`S4Squeeze.lean`) after `t` blocks +squeezed before: the states are permuted `t + n` times, and the buffers hold +bytes `168 t` to `168 (t + n)` of each seed's output. A squeeze writes only +below the saved registers (`sqT_ok`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Proof.MlKem.X86_64 (Keep ea_at) +open VG.Proof.MlKem.X86_64.S4 +open VG.Spec.MlKem (seed4 poly4) +open VG.Spec.Sha3 (keccakF RC) +open VG.Proof.Sha3 (byteOf iterF iterF_succ iterF_keccakF) +open VG.Proof.MlKem (xofByte) +open VG.Proof.Sha3.X86_64.X4 (la Lanes4 byte_of_lanes4 permute4_ok) + +/-- After `n` squeezes, `t` blocks after the absorption. -/ +structure SqT (σ : State) (t n : Nat) (s : State) : Prop where + env : Env σ s + r14 : s.gpr .r14 = 1 + rc : ∀ r < 24, ∀ k < 4, s.mem.readW (la (scr σ) (50 + r) k) 64 = RC r + lanes : Lanes4 s.mem (scr σ) fun k => iterF (t + n) (A0 (B σ k)) + buf : ∀ k < 4, ∀ p < 168 * n, s.mem (at' σ (oBuf + 504 * k + p)) = xofByte (B σ k) (168 * t + p) + +theorem sqT_of {σ s : State} (h : SqInv σ 0 s) : SqT σ 0 0 s := + ⟨h.env, h.r14, h.rc, h.lanes, fun _ _ p hp => absurd hp (by omega)⟩ + +/-- The low part of the scratch space, which the squeezes write. -/ +abbrev lowR (σ : State) : Region := ⟨scr σ, oSave⟩ + +/-- The permutation, after `n` squeezes. -/ +theorem permT_ok {σ : State} (hp : Pre σ) {t n : Nat} (hn : n < 3) {s : State} (h : SqT σ t n s) + {rest : Prog isa} {Q : State → Prop} (kont : ∀ s', Env σ s' ∧ s'.gpr .r14 = 1 ∧ + (∀ r < 24, ∀ k < 4, s'.mem.readW (la (scr σ) (50 + r) k) 64 = RC r) ∧ + Lanes4 s'.mem (scr σ) (fun k => iterF (t + n + 1) (A0 (B σ k))) ∧ + (∀ k < 4, ∀ p < 168 * n, s'.mem (at' σ (oBuf + 504 * k + p)) = xofByte (B σ k) (168 * t + p)) ∧ + Frame [lowR σ] s.mem s'.mem → WP isa rest s' Q) : + WP isa (.seq (.block permArgs) (.seq Impl.Sha3.X86_64.X4.permute4 rest)) s Q := by + refine WP.seq (WP.mono (args_ok h.env) fun s₁ ⟨⟨hm, hdi, hsi, hdx, hcx⟩, k₁⟩ => ?_) + have hrd : s₁.rd = σ.rd := k₁.2.1.trans h.env.rd + have hwr : s₁.wr = σ.wr := k₁.2.2.trans h.env.wr + refine WP.seq (WP.mono (permute4_ok (A := fun k => iterF (t + n) (A0 (B σ k))) + (pre4 hp hrd hwr (by rw [hm]; exact h.rc)) hdi hsi hdx (by rw [hcx, at', at', Offset.add_add]) + (by rw [hm]; exact h.lanes)) fun s₂ ⟨hl, hf, hrd₂, hwr₂, _, hg⟩ => kont s₂ ?_) + have hsub : ∀ r ∈ [(⟨scr σ, 800⟩ : Region), ⟨at' σ 800, 800⟩], Region.Sub r ⟨scr σ, oSave⟩ := by + intro r hr + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl + · exact Region.sub_prefix (by simp only [oSave]; omega) + · exact Offset.sub_base _ (by simp only [oSave]; omega) + refine ⟨Env.low (h.env.keep hm k₁ (by decide)) hsub hf hrd₂ hwr₂ fun r hr => hg r + (by simp only [List.mem_cons, List.not_mem_nil, or_false] at hr; rcases hr with rfl | rfl | rfl | rfl | rfl <;> decide) + (by simp only [List.mem_cons, List.not_mem_nil, or_false] at hr; rcases hr with rfl | rfl | rfl | rfl | rfl <;> decide), + by rw [hg _ (by decide) (by decide), k₁.gpr (by decide), h.r14], fun r hr k hk => ?_, ?_, fun k hk p hp' => ?_, + by rw [← hm]; exact hf.sub fun r hr => ⟨lowR σ, List.mem_singleton_self _, hsub r hr⟩⟩ + · rw [hf.readW (Region.contains_self _ _) (by + simpa using ⟨Offset.disjoint_base (scr σ) (k := 800) (d := 32 * (50 + r) + 8 * k) (n := 8) (by omega) (by omega), + Offset.disjoint (scr σ) (d := 32 * (50 + r) + 8 * k) (n := 8) (e := 800) (k := 800) (by omega) (by omega) + (by omega)⟩) (by decide), hm] + exact h.rc r hr k hk + · intro i hi k hk + rw [hl i hi k hk] + rfl + · rw [buf_frame (by simpa using ⟨Offset.disjoint_base (scr σ) (k := 800) (d := oBuf) (n := 2016) + (by simp only [oBuf]; omega) (by simp only [oBuf]; omega), Offset.disjoint (scr σ) (d := oBuf) (n := 2016) + (e := 800) (k := 800) (by simp only [oBuf]; omega) (by simp only [oBuf]; omega) (by omega)⟩) hf hk (by omega), hm] + exact h.buf k hk p hp' + +/-- During the copy of block `n`: the first `I` lanes of state `K` copied, +and all of the states before it. -/ +structure EXT (σ : State) (m₀ : Mem) (t n K I : Nat) (s : State) : Prop where + env : Env σ s + r14 : s.gpr .r14 = 1 + rc : ∀ r < 24, ∀ k < 4, s.mem.readW (la (scr σ) (50 + r) k) 64 = RC r + lanes : Lanes4 s.mem (scr σ) (fun k => iterF (t + n + 1) (A0 (B σ k))) + buf : ∀ k < 4, ∀ p < 504, (p < 168 * n ∨ (168 * n ≤ p ∧ p < 168 * n + 168 ∧ (k < K ∨ (k = K ∧ p < 168 * n + 8 * I)))) → + s.mem (at' σ (oBuf + 504 * k + p)) = xofByte (B σ k) (168 * t + p) + fr : Frame [lowR σ] m₀ s.mem + +open VG.Proof.Sha3.X86_64 (wp_movm wp_store wp_nil) in +theorem extT_step {σ : State} (hp : Pre σ) {m₀ : Mem} {t n K I : Nat} (hn : n < 3) (hK : K < 4) (hI : I < 21) + {s : State} (h : EXT σ m₀ t n K I s) : + WP isa (.block [.mov .rax (.mem (at_ .rbx (32 * I + 8 * K))), + .store (at_ .rbx (oBuf + 504 * K + 168 * n + 8 * I)) .rax]) s (EXT σ m₀ t n K (I + 1)) := by + refine wp_movm (a := at' σ (32 * I + 8 * K)) (by rw [ea_at, h.env.rbx]) + (in_scr' hp h.env.rd h.env.wr (by omega)) fun s₁ u₁ => wp_store (a := at' σ (oBuf + 504 * K + 168 * n + 8 * I)) + (by rw [ea_at, u₁.other _ (by decide), h.env.rbx]) (by rw [u₁.wr]; exact in_scr hp h.env.wr (by simp only [oBuf]; omega)) + fun s₂ g₂ m₂ r₂ w₂ => wp_nil ?_ + have hm : s₂.mem = s.mem.writeW (at' σ (oBuf + 504 * K + 168 * n + 8 * I)) (s.mem.readW (at' σ (32 * I + 8 * K)) 64) := by + rw [m₂, u₁.mem, u₁.gpr] + have hsub : Region.Sub ⟨at' σ (oBuf + 504 * K + 168 * n + 8 * I), 8⟩ (lowR σ) := + Offset.sub_base _ (by simp only [oBuf, oSave]; omega) + have hf : Frame [⟨at' σ (oBuf + 504 * K + 168 * n + 8 * I), 8⟩] s.mem s₂.mem := by + rw [hm]; exact (Frame.refl _ _).writeW (List.mem_singleton_self _) _ (Region.contains_self _ _) + refine ⟨Env.low h.env (fun r hr => by simp only [List.mem_singleton] at hr; subst hr; exact hsub) hf + (r₂.trans u₁.rd) (w₂.trans u₁.wr) + fun r hr => by + rw [g₂, u₁.other r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl <;> decide)], + by rw [g₂, u₁.other _ (by decide), h.r14], fun r hr k hk => ?_, fun i hi k hk => ?_, fun k hk p hp' hc => ?_, + h.fr.trans (hf.sub fun r hr => ⟨lowR σ, List.mem_singleton_self _, by + simp only [List.mem_singleton] at hr; subst hr; exact hsub⟩)⟩ + · have e := readW_writeW_off s.mem (scr σ) (s.mem.readW (at' σ (32 * I + 8 * K)) 64) (d := 32 * (50 + r) + 8 * k) + (e := oBuf + 504 * K + 168 * n + 8 * I) (n := 8) (by omega) (by simp only [oBuf]; omega) + (by simp only [oBuf]; omega) + rw [hm]; exact e.trans (h.rc r hr k hk) + · have e := readW_writeW_off s.mem (scr σ) (s.mem.readW (at' σ (32 * I + 8 * K)) 64) (d := 32 * i + 8 * k) + (e := oBuf + 504 * K + 168 * n + 8 * I) (n := 8) (by omega) (by simp only [oBuf]; omega) + (by simp only [oBuf]; omega) + rw [hm]; exact e.trans (h.lanes i hi k hk) + · rw [hm] + by_cases hw : k = K ∧ 168 * n + 8 * I ≤ p ∧ p < 168 * n + 8 * I + 8 + · obtain ⟨rfl, h₁, h₂⟩ := hw + rw [wb_in _ _ _ (by omega) (by simp only [oBuf]; omega) (by decide), + show 8 * (oBuf + 504 * k + p - (oBuf + 504 * k + 168 * n + 8 * I)) = 8 * (p - 168 * n - 8 * I) by omega, + byte_readW _ _ (by omega), at', Offset.add_add, + show 32 * I + 8 * k + (p - 168 * n - 8 * I) = 32 * ((8 * I + (p - 168 * n - 8 * I)) / 8) + 8 * k + + (8 * I + (p - 168 * n - 8 * I)) % 8 by omega, + byte_of_lanes4 h.lanes hk (by omega), show t + n + 1 = (t + n) + 1 from rfl, + ← xofByte_A0 (B_length σ k) (by omega), + show 168 * (t + n) + (8 * I + (p - 168 * n - 8 * I)) = 168 * t + p by omega] + · rw [wb_out _ _ _ (by simp only [oBuf]; omega) (by simp only [oBuf]; omega) (by simp only [oBuf]; omega)] + exact h.buf k hk p hp' (by omega) + +/-- Block `n` of each state's output. -/ +theorem extractT_ok {σ : State} (hp : Pre σ) {m₀ : Mem} {t n : Nat} (hn : n < 3) {s : State} + (h : EXT σ m₀ t n 0 0 s) : + WP isa (.block (extract n)) s fun s' => SqT σ t (n + 1) s' ∧ Frame [lowR σ] m₀ s'.mem := by + rw [extract_eq] + refine WP.mono (wp_range_flatMap (M := isa) (fun K s => EXT σ m₀ t n K 0 s) (fun K s hK h => ?_) 4 (Nat.le_refl _) + s h) fun s' h' => ⟨⟨h'.env, h'.r14, h'.rc, h'.lanes, fun k hk p hp' => h'.buf k hk p (by omega) (by omega)⟩, h'.fr⟩ + refine WP.mono (wp_range_flatMap (M := isa) (fun I s => EXT σ m₀ t n K I s) (fun I s hI h => extT_step hp hn hK hI h) + 21 (Nat.le_refl _) s h) fun s' h' => + ⟨h'.env, h'.r14, h'.rc, h'.lanes, fun k hk p hp' hc => h'.buf k hk p hp' (by omega), h'.fr⟩ + +/-- `squeeze4 n`: after `n + 1` squeezes; it writes only below the saved +registers. -/ +theorem sqT_ok {σ : State} (hp : Pre σ) {t n : Nat} (hn : n < 3) {s : State} (h : SqT σ t n s) : + WP isa (squeeze4 n) s fun s' => SqT σ t (n + 1) s' ∧ Frame [lowR σ] s.mem s'.mem := by + unfold squeeze4 + exact permT_ok hp hn h fun s' ⟨he, h14, hrc, hl, hb, hf⟩ => + extractT_ok hp hn ⟨he, h14, hrc, hl, fun k hk p _ hc => hb k hk p (by omega), hf⟩ + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Top.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Top.lean new file mode 100644 index 000000000..282e42ee0 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Top.lean @@ -0,0 +1,412 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Parse +import VerifiedGarbage.Proof.MlKem.X86_64.S4Top + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4_avx2`, correctness + +Untrusted: everything here is checked by Lean. The pieces, in order: the +prologue, the round constants and the padded seeds, and three squeezes (as +in `vg_mlkem_sample_ntt4_avx2`); the counts zeroed; the first half of each +seed (`P1`); three more squeezes; the second half of each seed, which +leaves the coefficients of its 1008 bytes of output and the AND of whether +each has 256 in `r14` (`P2`); and the epilogue, which returns it (`r4K`, as +`vg_mldsa_rej_ntt_poly`'s contract `rnK` for each seed). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlKem.X86_64.Sample4 +open VG.Impl.MlDsa.X86_64.Sample.Rej4 (oJ half first second zeroJ rejNTT4Avx2) +open VG.Proof.MlKem.X86_64 (Keep ea_at add_ofNat_zero pR WP.keep ofNat64_pred sample4K) +open VG.Proof.MlKem.X86_64.S4 +open VG.Proof.MlDsa.Sample (rnFold Stored stored_nil stored_frame stored_polyIs toPoly G_length) +open VG.Spec.MlKem (poly4 seed4) +open VG.Spec.MlDsa (Zq G PolyIs) +open VG.Proof.MlKem (xofByte) + +/-- `vg_mldsa_rej_ntt_poly4(seeds = rdi, a = rsi, scratch = rdx) -> eax` +(either implementation), with 24 bytes of stack below `rsp`: for each seed, +what `vg_mldsa_rej_ntt_poly`'s contract `rnK` says. -/ +def r4K : Contract isa where + pre := sample4K.pre + post s s' := + (s'.gpr .rax).setWidth 32 = + (if (List.range 4).all (fun k => (rnFold [] (G (seed4 s.mem (s.gpr .rdi) k) 1008)).length == 256) + then 1 else 0) ∧ + ∀ k < 4, (rnFold [] (G (seed4 s.mem (s.gpr .rdi) k) 1008)).length = 256 → + PolyIs s'.mem (poly4 (s.gpr .rsi) k) (toPoly (rnFold [] (G (seed4 s.mem (s.gpr .rdi) k) 1008))) + pub := sample4K.pub + +/-- The coefficients of seed `k`'s whole output. -/ +theorem Lt_336 (σ : State) (k : Nat) : Lt σ k 336 = rnFold [] (G (B σ k) 1008) := by + simp only [Lt]; rw [List.take_of_length_le (by rw [Xb, G_length])] + +theorem Lt_zero (σ : State) (k : Nat) : Lt σ k 0 = [] := by + simp only [Lt, Nat.mul_zero, List.take_zero]; rfl + +/-- 1 if each of the first `K` seeds has 256 coefficients, 0 otherwise. -/ +def okN (σ : State) (K : Nat) : Nat := if (List.range K).all (fun k => (Lt σ k 336).length == 256) then 1 else 0 + +/-- Where the counts are kept. -/ +abbrev jR (σ : State) : Region := ⟨at' σ oJ, 32⟩ + +/-- The saved registers. -/ +abbrev svR (σ : State) : Region := ⟨at' σ oSave, 40⟩ + +section +variable {σ : State} (hp : Pre σ) +include hp + +/-! ## The regions -/ + +omit hp in +theorem low_jR : (lowR σ).Disjoint (jR σ) := + Offset.base_disjoint (scr σ) (k := oSave) (e := oJ) (n := 32) (by simp only [oSave, oJ]; omega) + (by simp only [oJ]; omega) + +omit hp in +theorem sv_jR : (svR σ).Disjoint (jR σ) := + Offset.disjoint (scr σ) (d := oSave) (n := 40) (e := oJ) (k := 32) (.inl (by simp only [oSave, oJ]; omega)) + (by simp only [oSave]; omega) (by simp only [oJ]; omega) + +theorem low_poly {K : Nat} (hK : K < 4) : (lowR σ).Disjoint (pR (poly4 (aP σ) K)) := + Region.Disjoint.sub_right (Region.Disjoint.sub_left hp.a_scr.symm (Region.sub_prefix (by simp only [oSave]; omega))) + (sub_poly hK) + +theorem sv_poly {K : Nat} (hK : K < 4) : (svR σ).Disjoint (pR (poly4 (aP σ) K)) := + scr_poly hp hK (a := oSave) (n := 40) (by simp only [oSave]; omega) _ (List.mem_singleton_self _) + +theorem jR_poly {K : Nat} (hK : K < 4) : (jR σ).Disjoint (pR (poly4 (aP σ) K)) := + scr_poly hp hK (a := oJ) (n := 32) (by simp only [oJ]; omega) _ (List.mem_singleton_self _) + +theorem poly_jR (k : Nat) (hk : k < 4) : (VG.Proof.MlDsa.Sample.polyR (poly4 (aP σ) k)).Disjoint (jR σ) := + (jR_poly hp hk).symm + +/-! ## Frames -/ + +omit hp in +/-- `Env` after writes apart from the saved registers, in the regions of the precondition. -/ +theorem env_frame {s s' : State} (he : EnvK σ s) {rs : List Region} (hf : Frame rs s.mem s'.mem) + (hsv : ∀ r ∈ rs, (svR σ).Disjoint r) (hsub : ∀ r ∈ rs, ∃ R ∈ [aR σ, scrR σ, stkR σ], Region.Sub r R) + (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15], s'.gpr r = s.gpr r) : + EnvK σ s' := by + refine ⟨hrd.trans he.rd, hwr.trans he.wr, by rw [hg .rbx (by simp), he.rbx], by rw [hg .r12 (by simp), he.r12], + by rw [hg .r13 (by simp), he.r13], by rw [hg .rsp (by simp), he.rsp], by rw [hg .r15 (by simp), he.r15], + fun i hi => ?_, he.frame.trans (hf.sub hsub)⟩ + rw [hf.readW (Region.contains_self _ _) (fun r hr => (hsv r hr).sub_left + (Offset.sub (scr σ) (d := oSave + 8 * i) (n := 8) (e := oSave) (k := 40) (by omega) (by omega))) (by decide)] + exact he.saved i hi + +omit hp in +/-- `SqT` after writes apart from the low scratch space and the saved registers. -/ +theorem sqT_frame {t n : Nat} (hn : n ≤ 3) {s s' : State} (h : SqT σ t n s) {rs : List Region} (hf : Frame rs s.mem s'.mem) + (hlow : ∀ r ∈ rs, (lowR σ).Disjoint r) (hsv : ∀ r ∈ rs, (svR σ).Disjoint r) + (hsub : ∀ r ∈ rs, ∃ R ∈ [aR σ, scrR σ, stkR σ], Region.Sub r R) (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) + (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15, .r14], s'.gpr r = s.gpr r) : SqT σ t n s' := by + refine ⟨env_frame h.env hf hsv hsub hrd hwr fun r hr => hg r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr ⊢; rcases hr with h | h | h | h | h <;> simp [h]), + by rw [hg .r14 (by simp), h.r14], fun r hr k hk => ?_, fun i hi k hk => ?_, fun k hk p hp' => ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (fun r' hr' => (hlow r' hr').sub_left + (Offset.sub_base (scr σ) (d := 32 * (50 + r) + 8 * k) (by simp only [oSave]; omega))) (by decide)] + exact h.rc r hr k hk + · rw [hf.readW (Region.contains_self _ _) (fun r' hr' => (hlow r' hr').sub_left + (Offset.sub_base (scr σ) (d := 32 * i + 8 * k) (by simp only [oSave]; omega))) (by decide)] + exact h.lanes i hi k hk + · rw [frame_byte hf (R := ⟨at' σ (oBuf + 504 * k + p), 1⟩) (fun r' hr' => (hlow r' hr').sub_left + (Offset.sub_base (scr σ) (d := oBuf + 504 * k + p) (by simp only [oBuf, oSave]; omega))) + (Region.contains_self _ _)] + exact h.buf k hk p (by omega) + +/-- A count, apart from writes to a polynomial. -/ +theorem j_poly {K : Nat} (hK : K < 4) {m m' : Mem} (hf : Frame [pR (poly4 (aP σ) K)] m m') {k : Nat} (hk : k < 4) : + m'.readW (at' σ (oJ + 8 * k)) 64 = m.readW (at' σ (oJ + 8 * k)) 64 := + hf.readW (Region.contains_self _ _) (scr_poly hp hK (a := oJ + 8 * k) (n := 8) (by simp only [oJ]; omega)) + (by decide) + +omit hp in +/-- A stored polynomial, apart from writes to another one. -/ +theorem st_poly {K k : Nat} (hK : K < 4) (hk : k < 4) (hne : k ≠ K) {m m' : Mem} + (hf : Frame [pR (poly4 (aP σ) K)] m m') {L : List Zq} (h : Stored m (poly4 (aP σ) k) L) (hL : L.length ≤ 256) : + Stored m' (poly4 (aP σ) k) L := + stored_frame hf (poly_poly hK hk hne) h hL + +omit hp in +theorem sub_polyR {K : Nat} (hK : K < 4) : ∀ r ∈ [pR (poly4 (aP σ) K)], ∃ R ∈ [aR σ, scrR σ, stkR σ], Region.Sub r R := + fun r hr => by rw [List.mem_singleton.mp hr]; exact ⟨aR σ, by simp, sub_poly hK⟩ + +omit hp in +theorem sub_jR : ∀ r ∈ [jR σ], ∃ R ∈ [aR σ, scrR σ, stkR σ], Region.Sub r R := + fun r hr => by rw [List.mem_singleton.mp hr]; exact ⟨scrR σ, by simp, sub_scr (by simp only [oJ]; omega)⟩ + +omit hp in +theorem jR_contains {k : Nat} (hk : k < 4) : (jR σ).Contains (at' σ (oJ + 8 * k)) 8 := + Offset.contains (scr σ) (d := oJ + 8 * k) (n := 8) (e := oJ) (k := 32) (by omega) (by omega) (by simp only [oJ]; omega) + +omit hp in +theorem jR_write {k : Nat} (hk : k < 4) {m m' : Mem} (hf : Frame [jR σ] m m') (v : BitVec 64) : + Frame [jR σ] m (m'.writeW (at' σ (oJ + 8 * k)) v) := + hf.writeW (List.mem_singleton_self _) v (jR_contains hk) + +end + +/-! ## The first halves -/ + +/-- After the first half of the seeds before `K`. -/ +structure P1 (σ : State) (K : Nat) (s : State) : Prop where + sq : SqT σ 0 3 s + j : ∀ k < 4, s.mem.readW (at' σ (oJ + 8 * k)) 64 = BitVec.ofNat 64 (Lt σ k (if k < K then 168 else 0)).length + st : ∀ k < 4, Stored s.mem (poly4 (aP σ) k) (Lt σ k (if k < K then 168 else 0)) + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +theorem zeroJ_eq : zeroJ = [.mov32 .rax (.imm 0), .store (at_ .rbx (oJ + 8 * 0)) .rax, .store (at_ .rbx (oJ + 8 * 1)) .rax, + .store (at_ .rbx (oJ + 8 * 2)) .rax, .store (at_ .rbx (oJ + 8 * 3)) .rax] := rfl + +/-- The counts zeroed. -/ +theorem zeroJ_ok {s : State} (h : SqT σ 0 3 s) : WP isa (.block zeroJ) s (P1 σ 0) := by + have hin : ∀ k < 4, InRegions s.wr (at' σ (oJ + 8 * k)) 8 := fun k hk => + in_scr hp h.env.wr (by simp only [oJ]; omega) + rw [zeroJ_eq] + refine WP.mono (WP.keep [.rax] (Q := fun s' => s'.mem = (((s.mem.writeW (at' σ (oJ + 8 * 0)) (0 : BitVec 64)).writeW + (at' σ (oJ + 8 * 1)) (0 : BitVec 64)).writeW (at' σ (oJ + 8 * 2)) (0 : BitVec 64)).writeW (at' σ (oJ + 8 * 3)) + (0 : BitVec 64)) + (by + have h0 := hin 0 (by decide); have h1 := hin 1 (by decide); have h2 := hin 2 (by decide) + have h3 := hin 3 (by decide) + xrun [h.env.rbx, h0, h1, h2, h3] + rfl) + (by decide)) fun s' ⟨hm, k⟩ => ?_ + have hf : Frame [jR σ] s.mem s'.mem := by + rw [hm] + exact jR_write (by decide) (jR_write (by decide) (jR_write (by decide) (jR_write (by decide) (Frame.refl _ _) _) _) _) _ + refine ⟨sqT_frame (by decide) h hf (fun r hr => by rw [List.mem_singleton.mp hr]; exact low_jR) + (fun r hr => by rw [List.mem_singleton.mp hr]; exact sv_jR) sub_jR k.2.1 k.2.2 + fun r hr => k.gpr (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide), fun k hk => ?_, fun k hk => ?_⟩ + · rw [hm, ite_eq_right (by omega), Lt_zero, List.length_nil] + rcases (by omega : k = 0 ∨ k = 1 ∨ k = 2 ∨ k = 3) with rfl | rfl | rfl | rfl <;> + simp (disch := omega) only [oJ, Mem.readW_writeW_self64, rd64_off, Nat.reduceMul, Nat.reduceAdd] <;> rfl + · rw [ite_eq_right (by omega), Lt_zero] + exact stored_nil _ _ + +omit hp in +theorem hpre1 {K : Nat} (hK : K < 4) {s : State} (h : P1 σ K s) : HPre σ K 0 s := + ⟨h.sq.env, fun p hp' => by rw [h.sq.buf K hK p (by omega)], + by rw [h.j K hK, ite_eq_right (by omega), Nat.mul_zero], + by have := h.st K hK; rw [ite_eq_right (by omega)] at this; rwa [Nat.mul_zero]⟩ + +/-- `j` kept, after the first half of seed `K`. -/ +theorem firstEnd_ok {K : Nat} (hK : K < 4) {s s₁ : State} (h : P1 σ K s) (hA : HAt σ s K 0 168 s₁) : + WP isa (.block [.store (at_ .rbx (oJ + 8 * K)) .rdi]) s₁ (P1 σ (K + 1)) := by + have hin : InRegions s₁.wr (scr σ + BitVec.ofNat 64 (oJ + 8 * K)) 8 := in_scr hp hA.env.wr (by simp only [oJ]; omega) + refine WP.mono (WP.keep [] (Q := fun s' => s'.mem = s₁.mem.writeW (at' σ (oJ + 8 * K)) (s₁.gpr .rdi)) + (by xrun [hA.env.rbx, hin]) rfl) fun s₂ ⟨hm, k⟩ => ?_ + have hf₂ : Frame [jR σ] s₁.mem s₂.mem := by + rw [hm]; exact jR_write hK (Frame.refl _ _) _ + have k₁ : ∀ r ∈ [Reg.rbx, .r12, .r13, .rsp, .r15, .r14], s₁.gpr r = s.gpr r := fun r hr => + hA.kp.gpr (by simp only [List.mem_cons, List.not_mem_nil, or_false] at hr; rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide) + refine ⟨sqT_frame (by decide) (sqT_frame (by decide) h.sq hA.fr (fun r hr => by rw [List.mem_singleton.mp hr]; exact low_poly hp hK) + (fun r hr => by rw [List.mem_singleton.mp hr]; exact sv_poly hp hK) (sub_polyR hK) hA.kp.2.1 hA.kp.2.2 k₁) + hf₂ (fun r hr => by rw [List.mem_singleton.mp hr]; exact low_jR) + (fun r hr => by rw [List.mem_singleton.mp hr]; exact sv_jR) sub_jR k.2.1 k.2.2 fun r hr => k.gpr (by simp), + fun k hk => ?_, fun k hk => ?_⟩ + · rw [hm] + by_cases e : k = K + · subst e + rw [Mem.readW_writeW_self64, hA.rdi, ite_eq_left (by omega)] + · rw [rd64_off (by simp only [oJ]; omega) (by simp only [oJ]; omega) (by omega), j_poly hp hK hA.fr hk, h.j k hk] + congr 3 + by_cases hk' : k < K + · rw [ite_eq_left hk', ite_eq_left (by omega)] + · rw [ite_eq_right hk', ite_eq_right (by omega)] + · have hL := Lt_length_le σ k (if k < K + 1 then 168 else 0) + refine stored_frame hf₂ (fun r hr => by rw [List.mem_singleton.mp hr]; exact poly_jR hp k hk) ?_ hL + by_cases e : k = K + · subst e + rw [ite_eq_left (by omega)] + exact hA.stored + · have := st_poly hK hk e hA.fr (h.st k hk) (Lt_length_le σ k _) + by_cases hk' : k < K + · rw [ite_eq_left hk'] at this; rwa [ite_eq_left (by omega)] + · rw [ite_eq_right hk'] at this; rwa [ite_eq_right (by omega)] + +/-- The first half of seed `K`. -/ +theorem first_ok {K : Nat} (hK : K < 4) {s : State} (h : P1 σ K s) : WP isa (first K) s (P1 σ (K + 1)) := + WP.seq (WP.mono (half_ok hp hK (by decide) (hpre1 hK h)) fun _ hA => firstEnd_ok hp hK h hA) + +end + + +/-! ## The second halves -/ + +/-- After the second half of the seeds before `K`. -/ +structure P2 (σ : State) (K : Nat) (s : State) : Prop where + env : EnvK σ s + r14 : s.gpr .r14 = BitVec.ofNat 64 (okN σ K) + buf : ∀ k < 4, ∀ p < 504, s.mem (at' σ (oBuf + 504 * k + p)) = xofByte (B σ k) (504 + p) + j : ∀ k < 4, K ≤ k → s.mem.readW (at' σ (oJ + 8 * k)) 64 = BitVec.ofNat 64 (Lt σ k 168).length + st : ∀ k < 4, Stored s.mem (poly4 (aP σ) k) (Lt σ k (if k < K then 336 else 168)) + +theorem tail_ok (s : State) : + WP isa (.block [.mov .rax (.reg .rdi), .shift .shr .rax 8, .alu32 .and .r14 (.reg .rax)]) s fun s' => + (s'.gpr .r14 = BitVec.setWidth 64 ((s.gpr .r14).setWidth 32 &&& ((s.gpr .rdi) >>> 8).setWidth 32) ∧ + s'.mem = s.mem) ∧ Keep [.rax, .r14] s s' := by + refine WP.keep _ ?_ (by decide) + xrun + +/-- Whether a count is 256, from its bit 8. -/ +theorem full_bit {n : Nat} (hn : n ≤ 256) : + (BitVec.ofNat 64 n >>> 8).setWidth 32 = if n == 256 then 1 else 0 := by + by_cases e : n = 256 + · subst e; decide + · rw [ite_eq_right (by simpa using e)] + apply BitVec.eq_of_toNat_eq + have h0 : (0 : BitVec 32).toNat = 0 := rfl + simp only [BitVec.toNat_setWidth, BitVec.toNat_ushiftRight, BitVec.toNat_ofNat, Nat.shiftRight_eq_div_pow] + rw [h0] + omega + +theorem okN_succ (σ : State) (K : Nat) : BitVec.setWidth 64 ((BitVec.ofNat 64 (okN σ K)).setWidth 32 &&& + (BitVec.ofNat 64 (Lt σ K 336).length >>> 8).setWidth 32) = BitVec.ofNat 64 (okN σ (K + 1)) := by + rw [full_bit (Lt_length_le σ K 336)] + simp only [okN, List.range_succ, List.all_append, List.all_cons, List.all_nil, Bool.and_true] + cases (List.range K).all fun k => (Lt σ k 336).length == 256 <;> + cases (Lt σ K 336).length == 256 <;> rfl + +section +variable {σ : State} (hp : Pre σ) +include hp + +/-- During the second squeezes: the counts and coefficients of the first halves. -/ +structure M2 (σ : State) (n : Nat) (s : State) : Prop where + sq : SqT σ 3 n s + j : ∀ k < 4, s.mem.readW (at' σ (oJ + 8 * k)) 64 = BitVec.ofNat 64 (Lt σ k 168).length + st : ∀ k < 4, Stored s.mem (poly4 (aP σ) k) (Lt σ k 168) + +omit hp in +theorem m2_of {s : State} (h : P1 σ 4 s) : M2 σ 0 s := + ⟨⟨h.sq.env, h.sq.r14, h.sq.rc, h.sq.lanes, fun _ _ p hp' => absurd hp' (by omega)⟩, + fun k hk => by rw [h.j k hk, ite_eq_left hk], fun k hk => by have := h.st k hk; rwa [ite_eq_left hk] at this⟩ + +/-- A second squeeze. -/ +theorem sqM_ok {n : Nat} (hn : n < 3) {s : State} (h : M2 σ n s) : WP isa (squeeze4 n) s (M2 σ (n + 1)) := by + refine WP.mono (sqT_ok hp hn h.sq) fun s' ⟨q, hf⟩ => ⟨q, fun k hk => ?_, fun k hk => ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (fun r hr => by + rw [List.mem_singleton.mp hr] + exact (Offset.base_disjoint (scr σ) (k := oSave) (e := oJ + 8 * k) (n := 8) (by simp only [oSave, oJ]; omega) + (by simp only [oJ]; omega)).symm) (by decide), h.j k hk] + · exact stored_frame hf (fun r hr => by rw [List.mem_singleton.mp hr]; exact (low_poly hp hk).symm) (h.st k hk) + (Lt_length_le σ k _) + +omit hp in +/-- `vzeroupper`, after the second squeezes. -/ +theorem vz_ok {s : State} (h : M2 σ 3 s) : WP isa (.block [.vop .vzeroupper]) s (P2 σ 0) := by + refine WP.mono (WP.keep [] (Q := fun s' => s'.mem = s.mem) (by xrun; rfl) rfl) fun s' ⟨hm, k⟩ => ?_ + refine ⟨h.sq.env.keep hm k (by simp), by rw [k.gpr (by simp), h.sq.r14]; rfl, fun k hk p hp' => ?_, + fun k hk _ => by rw [hm]; exact h.j k hk, fun k hk => ?_⟩ + · rw [hm, h.sq.buf k hk p (by omega)] + · rw [ite_eq_right (by omega), hm]; exact h.st k hk + +omit hp in +theorem hpre2 {K : Nat} (hK : K < 4) {s : State} (h : P2 σ K s) : HPre σ K 1 s := + ⟨h.env, fun p hp' => by rw [h.buf K hK p hp'], by rw [h.j K hK (Nat.le_refl _), Nat.mul_one], + by have := h.st K hK; rw [ite_eq_right (by omega)] at this; rwa [Nat.mul_one]⟩ + +/-- The AND of whether seed `K` has 256 coefficients, after its second half. -/ +theorem secondEnd_ok {K : Nat} (hK : K < 4) {s s₁ : State} (h : P2 σ K s) (hA : HAt σ s K 1 168 s₁) : + WP isa (.block [.mov .rax (.reg .rdi), .shift .shr .rax 8, .alu32 .and .r14 (.reg .rax)]) s₁ (P2 σ (K + 1)) := by + refine WP.mono (tail_ok s₁) fun s₂ ⟨⟨h14, hm⟩, k⟩ => ?_ + have hf : Frame [pR (poly4 (aP σ) K)] s.mem s₂.mem := by rw [hm]; exact hA.fr + have hrdi : s₁.gpr .rdi = BitVec.ofNat 64 (Lt σ K 336).length := hA.rdi + refine ⟨hA.env.keep hm k (by simp), ?_, fun k hk p hp' => ?_, fun k hk hKk => ?_, fun k hk => ?_⟩ + · rw [h14, hA.kp.gpr (by decide), h.r14, hrdi, okN_succ] + · rw [frame_byte hf (R := ⟨at' σ (oBuf + 504 * k + p), 1⟩) (scr_poly hp hK (by simp only [oBuf]; omega)) + (Region.contains_self _ _)] + exact h.buf k hk p hp' + · rw [hm, j_poly hp hK hA.fr hk, h.j k hk (by omega)] + · rw [hm] + by_cases e : k = K + · subst e + rw [ite_eq_left (by omega)] + exact hA.stored + · have := st_poly hK hk e hA.fr (h.st k hk) (Lt_length_le σ k _) + by_cases hk' : k < K + · rw [ite_eq_left hk'] at this; rwa [ite_eq_left (by omega)] + · rw [ite_eq_right hk'] at this; rwa [ite_eq_right (by omega)] + +/-- The second half of seed `K`. -/ +theorem second_ok {K : Nat} (hK : K < 4) {s : State} (h : P2 σ K s) : WP isa (second K) s (P2 σ (K + 1)) := + WP.seq (WP.mono (half_ok hp hK (by decide) (hpre2 hK h)) fun _ hA => secondEnd_ok hp hK h hA) + +/-- The return value, and the callee-saved registers restored. -/ +theorem end_ok {s : State} (h : P2 σ 4 s) : + WP isa (.block epi) s fun s' => r4K.post σ s' ∧ gprPreserved σ s' := by + have hin : ∀ i < 5, InRegions (s.rd ++ s.wr) (scr σ + BitVec.ofNat 64 (oSave + 8 * i)) 8 := fun i hi => + in_scr' hp h.env.rd h.env.wr (by simp only [oSave]; omega) + rw [epi_eq] + refine WP.mono (WP.keep [.rax, .r14, .r13, .r12, .rbp, .rbx] (Q := fun s' => s'.mem = s.mem ∧ + (s'.gpr .rax).setWidth 32 = (s.gpr .r14).setWidth 32 ∧ + s'.gpr .r14 = s.mem.readW (at' σ (oSave + 8 * 4)) 64 ∧ s'.gpr .r13 = s.mem.readW (at' σ (oSave + 8 * 3)) 64 ∧ + s'.gpr .r12 = s.mem.readW (at' σ (oSave + 8 * 2)) 64 ∧ s'.gpr .rbp = s.mem.readW (at' σ (oSave + 8 * 1)) 64 ∧ + s'.gpr .rbx = s.mem.readW (at' σ (oSave + 8 * 0)) 64) + (by + have h0 := hin 0 (by decide); have h1 := hin 1 (by decide); have h2 := hin 2 (by decide) + have h3 := hin 3 (by decide); have h4 := hin 4 (by decide) + simp only [oSave, Nat.reduceMul, Nat.reduceAdd] at h0 h1 h2 h3 h4 + xrun [h0, h1, h2, h3, h4, h.env.rbx] + exact ⟨rfl, rfl, rfl, rfl, rfl⟩) + (by decide)) fun s' ⟨⟨hm, hax, h14, h13, h12, hbp, hbx⟩, k⟩ => ?_ + refine ⟨⟨?_, fun k hk e => ?_⟩, fun r hr => ?_, ?_⟩ + · rw [hax, h.r14, okN] + simp only [Lt_336] + split <;> rfl + · rw [hm] + have := h.st k hk + rw [ite_eq_left hk, Lt_336] at this + exact stored_polyIs this e + · simp only [calleeSaved, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl | rfl + · rw [hbx]; exact h.env.saved 0 (by decide) + · rw [hbp]; exact h.env.saved 1 (by decide) + · rw [k.gpr (by decide)]; exact h.env.rsp + · rw [h12]; exact h.env.saved 2 (by decide) + · rw [h13]; exact h.env.saved 3 (by decide) + · rw [h14]; exact h.env.saved 4 (by decide) + · rw [k.gpr (by decide)]; exact h.env.r15 + · rw [hm] + exact h.env.frame.readW (Region.contains_self _ _) (by + simpa using ⟨hp.ret_a, hp.ret_scr, Offset.base_disjoint_below (σ.gpr .rsp) (n := 24) (k := 8) (by omega)⟩) + (by decide) + +/-- Everything after the prologue, the round constants and the padded seeds. -/ +theorem body_ok {s : State} (h : SqInv σ 0 s) : + WP isa (.seq (squeeze4 0) (.seq (squeeze4 1) (.seq (squeeze4 2) (.seq (.block zeroJ) + (.seq (first 0) (.seq (first 1) (.seq (first 2) (.seq (first 3) + (.seq (squeeze4 0) (.seq (squeeze4 1) (.seq (squeeze4 2) (.seq (.block [.vop .vzeroupper]) + (.seq (second 0) (.seq (second 1) (.seq (second 2) (.seq (second 3) (.block epi))))))))))))))))) s + fun s' => r4K.post σ s' ∧ gprPreserved σ s' := by + refine WP.seq (WP.mono (sqT_ok hp (by decide) (sqT_of h)) fun s₁ ⟨q₁, _⟩ => WP.seq (WP.mono + (sqT_ok hp (by decide) q₁) fun s₂ ⟨q₂, _⟩ => WP.seq (WP.mono (sqT_ok hp (by decide) q₂) fun s₃ ⟨q₃, _⟩ => + WP.seq (WP.mono (zeroJ_ok hp q₃) fun s₄ p₀ => ?_)))) + refine WP.seq (WP.mono (first_ok hp (by decide) p₀) fun _ p₁ => WP.seq (WP.mono (first_ok hp (by decide) p₁) + fun _ p₂ => WP.seq (WP.mono (first_ok hp (by decide) p₂) fun _ p₃ => + WP.seq (WP.mono (first_ok hp (by decide) p₃) fun _ p₄ => ?_)))) + refine WP.seq (WP.mono (sqM_ok hp (by decide) (m2_of p₄)) fun _ m₁ => WP.seq (WP.mono (sqM_ok hp (by decide) m₁) + fun _ m₂ => WP.seq (WP.mono (sqM_ok hp (by decide) m₂) fun _ m₃ => WP.seq (WP.mono (vz_ok m₃) fun _ r₀ => ?_)))) + exact WP.seq (WP.mono (second_ok hp (by decide) r₀) fun _ r₁ => WP.seq (WP.mono (second_ok hp (by decide) r₁) + fun _ r₂ => WP.seq (WP.mono (second_ok hp (by decide) r₂) fun _ r₃ => + WP.seq (WP.mono (second_ok hp (by decide) r₃) fun _ r₄ => end_ok hp r₄)))) + +end + +theorem correct (σ : State) (hs : r4K.pre σ) : + ∃ t s', Exec isa rejNTT4Avx2 σ t s' ∧ abiPreserved σ s' ∧ r4K.post σ s' := by + have hp := pre_of hs + obtain ⟨t, s', he, hF⟩ := WP.seq (WP.mono (start_ok hp) fun _ h => body_ok hp h) + exact ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he hF.2, hF.1⟩ + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Verified.lean new file mode 100644 index 000000000..ddb0be67b --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/Rej4Verified.lean @@ -0,0 +1,79 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Scalar +import VerifiedGarbage.Proof.MlKem.X86_64.S4Verified + +/-! +# ML-DSA on x86-64: `vg_mldsa_rej_ntt_poly4` and `vg_mldsa_rej_ntt_poly4_avx2`, verified + +Untrusted: everything here is checked by Lean. The contract of the proofs +(`r4K`) implies the shared one of `Spec/`: a seed with 256 coefficients in +the 1008 bytes both implementations sample from has them within those +bounds (`rejNTT_some`), and one without has none within the least bound +(`rejNTT_none`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Rej4 + +open VG VG.X86_64 +open VG.Proof.MlDsa.Sample (rnFold rejNTT_some rejNTT_none) +open VG.Proof.MlDsa.X86_64.Sample (leakBytes_inj) +open VG.Spec.MlDsa (G) + +theorem seed4_eq : Spec.MlDsa.seed4 = Spec.MlKem.seed4 := rfl +theorem poly4_eq : Spec.MlDsa.poly4 = Spec.MlKem.poly4 := rfl + +theorem r4_post {s s' : State} (h : r4K.post s s') : + let r := (s'.gpr .rax).setWidth 32 + (r = 1 → ∀ k < 4, Spec.MlDsa.Reduced s'.mem (Spec.MlDsa.poly4 (s.gpr .rsi) k)) ∧ + ((r = 1 ∧ ∀ k < 4, ∃ b : Spec.MlDsa.Bounds, Spec.MlDsa.rejNTTPoly b.rejNTT + (Spec.MlDsa.seed4 s.mem (s.gpr .rdi) k) = some (Spec.MlDsa.polyAt s'.mem (Spec.MlDsa.poly4 (s.gpr .rsi) k))) ∨ + (r = 0 ∧ ∃ k < 4, Spec.MlDsa.rejNTTPoly Spec.MlDsa.minBounds.rejNTT + (Spec.MlDsa.seed4 s.mem (s.gpr .rdi) k) = none)) := by + obtain ⟨hr, hp⟩ := h + intro r + simp only [seed4_eq, poly4_eq] + by_cases hall : ((List.range 4).all fun k => + (rnFold [] (G (Spec.MlKem.seed4 s.mem (s.gpr .rdi) k) 1008)).length == 256) = true + · rw [ite_eq_left hall] at hr + have hs : ∀ k < 4, (rnFold [] (G (Spec.MlKem.seed4 s.mem (s.gpr .rdi) k) 1008)).length = 256 := fun k hk => by + simpa using List.all_eq_true.mp hall k (List.mem_range.mpr hk) + refine ⟨fun _ k hk => (hp k hk (hs k hk)).1, .inl ⟨hr, fun k hk => ⟨{ Spec.MlDsa.minBounds with rejNTT := 1008 }, ?_⟩⟩⟩ + show Spec.MlDsa.rejNTTPoly 1008 _ = _ + rw [rejNTT_some (hs k hk), (hp k hk (hs k hk)).2] + · rw [ite_eq_right hall] at hr + refine ⟨fun h1 => absurd (hr.symm.trans h1) (by decide), .inr ⟨hr, ?_⟩⟩ + simp only [List.all_eq_true, List.mem_range, not_forall, beq_iff_eq] at hall + obtain ⟨k, hk, hk'⟩ := hall + exact ⟨k, hk, rejNTT_none (B := 1008) (by decide) (by decide) hk'⟩ + +theorem rej4_verified (c : Prog isa) (hc : ∀ σ, r4K.pre σ → ∃ t s', Exec isa c σ t s' ∧ abiPreserved σ s' ∧ r4K.post σ s') + (ht : ConstantTime isa r4K.pre r4K.pub c) : Verified X86_64.target c (Spec.MlDsa.rejNTT4Contract X86_64.abi 24) := + Verified.of_correct hc ht + { pre := by sig_implies_pre [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, + X86_64.abi, X86_64.argRegs] + post := by + intro s s' _ h + sig_post [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, X86_64.abi, X86_64.argRegs] + exact r4_post h + pub := by + intro s₁ s₂ _ _ h + sig_pub [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, X86_64.abi, + X86_64.argRegs] at h + sig_split h + sig_reduce [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, X86_64.abi, + X86_64.argRegs] + sig_simp [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, X86_64.abi, + X86_64.argRegs] [Nat.forall_lt_succ_right, Nat.not_lt_zero, false_imp_iff, forall_const, true_and] + sig_and_intros + sig_close + all_goals first | with_reducible assumption | exact leakBytes_inj ‹_› + sat := by + sig_implies_sat [Spec.MlDsa.rejNTT4Contract, Spec.MlDsa.rejNTT4Sig, r4K, MlKem.X86_64.sample4K, X86_64.abi, + X86_64.argRegs] [MlKem.X86_64.sample4Sat] using MlKem.X86_64.sample4Sat } + +theorem rejNTT4Avx2_verified : Verified X86_64.target Impl.MlDsa.X86_64.Sample.Rej4.rejNTT4Avx2 + (Spec.MlDsa.rejNTT4Contract X86_64.abi 24) := rej4_verified _ correct ct + +theorem rejNTT4_verified : Verified X86_64.target Impl.MlDsa.X86_64.Sample.Rej4.rejNTT4 + (Spec.MlDsa.rejNTT4Contract X86_64.abi 24) := rej4_verified _ correct_scalar ct_scalar + +end VG.Proof.MlDsa.X86_64.Rej4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBounded.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBounded.lean index 133c1036a..be850a2d1 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBounded.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBounded.lean @@ -117,7 +117,7 @@ theorem lat_step {t : Nat} {s : State} (ht : t < 544) (h : LAt σ t s) : WP isa (rbBody (etaOf σ)) s fun s' => LAt σ (t + 1) s' ∧ s'.zf = some (BitVec.ofNat 64 (544 - t) - 1 == 0) := by have hw : pR (σ.gpr .rdx) ∈ s.wr := by rw [h.env.wr, hp.2.1]; simp have hp' := spOk hp - refine WP.mono (rbBody_ok (eta hp) s (aP := σ.gpr .rdx) h.env.rbp h.rdi (Lt_length_le t) hw h.stored + refine WP.mono (rbBody_ok (eta hp) s (aP := σ.gpr .rdx) h.env.rbp h.rdi (Lt_length_le t) (.of_mem hw) h.stored (by rw [h.rsi, at_add]; exact inScrRd hp' h.env (by omega))) fun s' ⟨hdi, hst, hf, hsi, hcx, hz, hk⟩ => ?_ have ht1 : Lt σ (t + 1) = rbStep (etaOf σ) (Lt σ t) ((X σ).getD t 0) := by simp only [Lt] diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedCT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedCT.lean index a77f4b731..a3efd7896 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedCT.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedCT.lean @@ -117,7 +117,7 @@ structure BPre (s : State) (aP : Addr) (L : List Zq) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length ≤ 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L r0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 1 @@ -132,7 +132,7 @@ structure MPre (s : State) (aP : Addr) (L : List Zq) (z : Byte) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length ≤ 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L cf : s.cf = some (decide (L.length < 256)) rax : s.gpr .rax = BitVec.setWidth 64 z @@ -167,7 +167,7 @@ theorem load_ct (η : Nat) : RelCT isa (BRel η) (.block rbLoad) (R1 η) := by def R2 (η : Nat) (s₁ s₂ : State) : Prop := ∃ (aP : Addr) (L₁ L₂ : List Zq) (z₁ z₂ : Byte), L₁.length = L₂.length ∧ L₁.length ≤ 256 ∧ L₂.length ≤ 256 ∧ s₁.gpr .rbp = aP ∧ s₂.gpr .rbp = aP ∧ s₁.gpr .rdi = BitVec.ofNat 64 L₁.length ∧ - s₂.gpr .rdi = BitVec.ofNat 64 L₂.length ∧ pR aP ∈ s₁.wr ∧ pR aP ∈ s₂.wr ∧ Stored s₁.mem aP L₁ ∧ + s₂.gpr .rdi = BitVec.ofNat 64 L₂.length ∧ CoeffsWr s₁.wr aP ∧ CoeffsWr s₂.wr aP ∧ Stored s₁.mem aP L₁ ∧ Stored s₂.mem aP L₂ ∧ s₁.gpr .rax = BitVec.setWidth 64 z₁ ∧ s₂.gpr .rax = BitVec.setWidth 64 z₂ ∧ halfByteOk η (z₁.toNat / 16) = halfByteOk η (z₂.toNat / 16) ∧ s₁.gpr .rcx = s₂.gpr .rcx ∧ s₁.gpr .rsi = s₂.gpr .rsi @@ -176,7 +176,7 @@ def R2 (η : Nat) (s₁ s₂ : State) : Prop := def R3 (η : Nat) (s₁ s₂ : State) : Prop := ∃ aP L₁ L₂ b₁ b₂, L₁.length = L₂.length ∧ L₁.length ≤ 256 ∧ s₁.gpr .rbp = aP ∧ s₂.gpr .rbp = aP ∧ s₁.gpr .rdi = BitVec.ofNat 64 L₁.length ∧ - s₂.gpr .rdi = BitVec.ofNat 64 L₂.length ∧ pR aP ∈ s₁.wr ∧ pR aP ∈ s₂.wr ∧ Stored s₁.mem aP L₁ ∧ + s₂.gpr .rdi = BitVec.ofNat 64 L₂.length ∧ CoeffsWr s₁.wr aP ∧ CoeffsWr s₂.wr aP ∧ Stored s₁.mem aP L₁ ∧ Stored s₂.mem aP L₂ ∧ b₁ < 16 ∧ b₂ < 16 ∧ (s₁.gpr .rdx).setWidth 32 = BitVec.ofNat 32 b₁ ∧ (s₂.gpr .rdx).setWidth 32 = BitVec.ofNat 32 b₂ ∧ halfByteOk η b₁ = halfByteOk η b₂ ∧ s₁.cf = some (decide (L₁.length < 256)) ∧ s₂.cf = some (decide (L₂.length < 256)) ∧ @@ -333,7 +333,7 @@ def LI (η n : Nat) (s₁ s₂ : State) : Prop := theorem bpre {σ : State} (hp : rbK.pre σ) {t : Nat} (ht : t < 544) {s : State} (h : LAt σ t s) : BPre s (σ.gpr .rdx) (Lt σ t) := - ⟨h.env.rbp, h.rdi, Lt_length_le t, by rw [h.env.wr, hp.2.1]; simp, h.stored, + ⟨h.env.rbp, h.rdi, Lt_length_le t, .of_mem (by rw [h.env.wr, hp.2.1]; simp), h.stored, by rw [h.rsi, at_add]; exact inScrRd (spOk hp) h.env (by omega)⟩ theorem li_brel {η n : Nat} {s₁ s₂ : State} (h : LI η n s₁ s₂) : BRel η s₁ s₂ := by diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedLoop.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedLoop.lean index cdf007305..1342c5570 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedLoop.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejBoundedLoop.lean @@ -152,7 +152,7 @@ theorem hbTry_length_le {η : Nat} {L : List Zq} (hL : L.length < 256) (b : Nat) /-- The second try, if `j < 256`. -/ theorem rbMid2_ok {η : Nat} (hη : η = 2 ∨ η = 4) (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (hcf : s.cf = some (decide ((s.gpr .rdi).toNat < 256))) {b : Nat} (hb : b < 16) (hdx : (s.gpr .rdx).setWidth 32 = BitVec.ofNat 32 b) : WP isa (.ite .b (rbTry η) (.block [])) s fun s' => @@ -169,7 +169,7 @@ theorem rbMid2_ok {η : Nat} (hη : η = 2 ∨ η = 4) (s : State) {aP : Addr} { /-- The two tries, if `j < 256`. -/ theorem rbMid_ok {η : Nat} (hη : η = 2 ∨ η = 4) (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (hcf : s.cf = some (decide ((s.gpr .rdi).toNat < 256))) {z : Byte} (hax : s.gpr .rax = BitVec.setWidth 64 z) (hdx : (s.gpr .rdx).setWidth 32 = BitVec.ofNat 32 (z.toNat % 16)) : WP isa (.ite .b (.seq (rbTry η) (.seq (.block rbHi) (.ite .b (rbTry η) (.block [])))) (.block [])) s @@ -196,7 +196,7 @@ theorem sx1' : BitVec.signExtend 64 (1 : BitVec 32) = BitVec.ofNat 64 1 := by de /-- An iteration: what `rbStep` does to the coefficients `L`. -/ theorem rbBody_ok {η : Nat} (hη : η = 2 ∨ η = 4) (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 1) : WP isa (rbBody η) s fun s' => s'.gpr .rdi = BitVec.ofNat 64 (rbStep η L (s.mem (s.gpr .rsi))).length ∧ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNtt.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNtt.lean index fe55b6adf..341329234 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNtt.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNtt.lean @@ -112,7 +112,7 @@ theorem Lt_length_le (t : Nat) : (Lt σ t).length ≤ 256 := rnFold_length_le (b theorem lat_step {t : Nat} {s : State} (ht : t < 336) (h : LAt σ t s) : WP isa rnBody s fun s' => LAt σ (t + 1) s' ∧ s'.zf = some (BitVec.ofNat 64 (336 - t) - 1 == 0) := by have hw : pR (σ.gpr .rsi) ∈ s.wr := by rw [h.env.wr, hp.2.1]; simp - refine WP.mono (rnBody_ok s (aP := σ.gpr .rsi) h.env.rbp h.rdi (Lt_length_le t) hw h.stored + refine WP.mono (rnBody_ok s (aP := σ.gpr .rsi) h.env.rbp h.rdi (Lt_length_le t) (.of_mem hw) h.stored (by simpa using lat_regions hp h (k := 0) (by omega)) (lat_regions hp h (by omega)) (lat_regions hp h (by omega))) fun s' ⟨hdi, hst, hf, hsi, hcx, hz, hk⟩ => ?_ have e0 := out_byte h (k := 0) (by omega) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttCT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttCT.lean index a8f88337d..ff749eb6f 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttCT.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttCT.lean @@ -57,7 +57,7 @@ structure MPre (s : State) (aP : Addr) (L : List Zq) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length ≤ 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L cf : s.cf = some (decide ((s.gpr .rdi).toNat < 256)) @@ -127,7 +127,7 @@ structure BPre (s : State) (aP : Addr) (L : List Zq) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length ≤ 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L r0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 1 r1 : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 1) 1 @@ -196,7 +196,7 @@ def LI (n : Nat) (s₁ s₂ : State) : Prop := theorem bpre {σ : State} (hp : rnK.pre σ) {t : Nat} (ht : t < 336) {s : State} (h : LAt σ t s) : BPre s (σ.gpr .rsi) (Lt σ t) := - ⟨h.env.rbp, h.rdi, Lt_length_le t, by rw [h.env.wr, hp.2.1]; simp, h.stored, + ⟨h.env.rbp, h.rdi, Lt_length_le t, .of_mem (by rw [h.env.wr, hp.2.1]; simp), h.stored, by simpa using lat_regions hp h (k := 0) (by omega), lat_regions hp h (by omega), lat_regions hp h (by omega)⟩ theorem li_brel {n : Nat} {s₁ s₂ : State} (h : LI n s₁ s₂) : BRel s₁ s₂ := by diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttLoop.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttLoop.lean index f0b9a5d52..c782b78b3 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttLoop.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/RejNttLoop.lean @@ -68,14 +68,21 @@ theorem storeJ_ok (r : Reg) (s : State) {a : Addr} (ha : s.ea aJ = a) (hw : InRe refine WP.keep _ ?_ (by rfl) xrun [ha, hw] +/-- Each coefficient of the polynomial at `aP` lies in one of the writable +regions `wr`. -/ +def CoeffsWr (wr : List Region) (aP : Addr) : Prop := ∀ i < 256, InRegions wr (coeffAddr aP i) 4 + +theorem CoeffsWr.of_mem {wr : List Region} {aP : Addr} (h : pR aP ∈ wr) : CoeffsWr wr aP := + fun _ hi => ⟨_, h, coeff_contains _ hi⟩ + /-- Storing the word `v` (less than `q`) as the next coefficient. -/ theorem store_next {s : State} {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length < 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length < 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (r : Reg) {v : BitVec 32} (hv : (s.gpr r).setWidth 32 = v) (hq : v.toNat < q) : WP isa (.block [.store32 aJ r, .alu .add .rdi (.imm 1)]) s fun s' => s'.gpr .rdi = BitVec.ofNat 64 (L ++ [Fin.ofNat q v.toNat]).length ∧ Stored s'.mem aP (L ++ [Fin.ofNat q v.toNat]) ∧ Frame [pR aP] s.mem s'.mem ∧ Keep [.rdi] s s' := by - refine WP.mono (storeJ_ok r s (ea_aJ s hbp hdi) ⟨_, hw, coeff_contains _ hL⟩) fun s' ⟨⟨hm, hdi'⟩, k⟩ => ?_ + refine WP.mono (storeJ_ok r s (ea_aJ s hbp hdi) (hw _ hL)) fun s' ⟨⟨hm, hdi'⟩, k⟩ => ?_ have hz : zw (Fin.ofNat q v.toNat) = v := by apply BitVec.eq_of_toNat_eq rw [zw_toNat, Fin.val_ofNat, Nat.mod_eq_of_lt hq] @@ -90,7 +97,7 @@ structure TryPre (s : State) (aP : Addr) (L : List Zq) : Prop where rbp : s.gpr .rbp = aP rdi : s.gpr .rdi = BitVec.ofNat 64 L.length len : L.length < 256 - wr : pR aP ∈ s.wr + wr : CoeffsWr s.wr aP st : Stored s.mem aP L /-- The coefficients after a try of the value `v`. -/ @@ -124,7 +131,7 @@ theorem rnStep_eq (L : List Zq) (b₀ b₁ b₂ : Byte) : rnStep L b₀ b₁ b /-- The try, if `j < 256`. -/ theorem rnMid_ok (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (hcf : s.cf = some (decide ((s.gpr .rdi).toNat < 256))) : WP isa (.ite .b rnTry (.block [])) s fun s' => s'.gpr .rdi = BitVec.ofNat 64 (rnMid L ((s.gpr .r8).setWidth 32).toNat).length ∧ @@ -147,7 +154,7 @@ theorem sx3 : BitVec.signExtend 64 (3 : BitVec 32) = BitVec.ofNat 64 3 := by dec /-- An iteration: what `rnStep` does to the coefficients `L`. -/ theorem rnBody_ok (s : State) {aP : Addr} {L : List Zq} (hbp : s.gpr .rbp = aP) - (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : pR aP ∈ s.wr) + (hdi : s.gpr .rdi = BitVec.ofNat 64 L.length) (hL : L.length ≤ 256) (hw : CoeffsWr s.wr aP) (hst : Stored s.mem aP L) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rsi) 1) (h1 : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 1) 1) (h2 : InRegions (s.rd ++ s.wr) (s.gpr .rsi + BitVec.ofNat 64 2) 1) : diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Base.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Base.lean index a1666ba22..46f211eed 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Base.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Base.lean @@ -20,7 +20,7 @@ Untrusted: everything here is checked by Lean. checked by evaluation (`inB`, `sepB`). * What a piece of code leaves (`PostB`): the permissions, the registers of the layout and the stack pointer, and memory but within the regions it - writes and the 24 bytes of stack below `rsp` (its calls' return + writes and the 32 bytes of stack below `rsp` (its calls' return addresses). * A call of verified code, with the moves of its arguments before it (`callAt_ok`, `callAt_tr`). @@ -121,7 +121,7 @@ structure Post (s s' : State) (W : List Region) : Prop where rd : s'.rd = s.rd wr : s'.wr = s.wr cs : ∀ r ∈ calleeSaved, s'.gpr r = s.gpr r - frame : Frame (W ++ [below (s.gpr .rsp) 24]) s.mem s'.mem + frame : Frame (W ++ [below (s.gpr .rsp) 32]) s.mem s'.mem /-- What a piece of code leaves: the permissions, the registers `bases` and the stack pointer, and memory but within `W` and the stack. -/ @@ -130,7 +130,7 @@ structure PostB (s s' : State) (W : List Region) : Prop where wr : s'.wr = s.wr bs : ∀ r ∈ bases, s'.gpr r = s.gpr r rsp : s'.gpr .rsp = s.gpr .rsp - frame : Frame (W ++ [below (s.gpr .rsp) 24]) s.mem s'.mem + frame : Frame (W ++ [below (s.gpr .rsp) 32]) s.mem s'.mem theorem Post.rsp {s s' : State} {W : List Region} (h : Post s s' W) : s'.gpr .rsp = s.gpr .rsp := h.cs .rsp (by decide) @@ -259,13 +259,13 @@ structure Lay (rbs wbs : List (Reg × Nat)) (s : State) : Prop where small : ∀ b ∈ rbs ++ wbs, b.2 < 2 ^ 31 dj : ∀ b ∈ rbs ++ wbs, ∀ b' ∈ rbs ++ wbs, b.1 ≠ b'.1 → (b.1 ∈ wRegs ∨ b'.1 ∈ wRegs) → Region.Disjoint ⟨s.gpr b.1, b.2⟩ ⟨s.gpr b'.1, b'.2⟩ - stk : ∀ b ∈ rbs ++ wbs, (below (s.gpr .rsp) 24).Disjoint ⟨s.gpr b.1, b.2⟩ + stk : ∀ b ∈ rbs ++ wbs, (below (s.gpr .rsp) 32).Disjoint ⟨s.gpr b.1, b.2⟩ nw : ∀ b ∈ rbs ++ wbs, (s.gpr b.1).toNat + b.2 ≤ 2 ^ 64 rd : ∀ b ∈ rbs ++ wbs, InRegions (s.rd ++ s.wr) (s.gpr b.1) b.2 wr : ∀ b ∈ wbs, InRegions s.wr (s.gpr b.1) b.2 ret : ∀ b ∈ rbs ++ wbs, (Region.mk (s.gpr .rsp) 8).Disjoint ⟨s.gpr b.1, b.2⟩ bs : ∀ b ∈ rbs ++ wbs, b.1 ∈ bases - sp24 : 24 ≤ (s.gpr .rsp).toNat + sp32 : 32 ≤ (s.gpr .rsp).toNat section variable {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) @@ -292,7 +292,7 @@ theorem Lay.disj {p q : Ptr} {l k : Nat} (h : sepB (rbs ++ wbs) p l q k = true) · exact (off_disj h2 (by omega)).symm theorem Lay.stkD {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : - (below (s.gpr .rsp) 24).Disjoint ⟨pa s p, l⟩ := by + (below (s.gpr .rsp) 32).Disjoint ⟨pa s p, l⟩ := by obtain ⟨n, hn, hsub⟩ := L.sub h exact (L.stk _ hn).sub_right hsub @@ -327,7 +327,7 @@ theorem Lay.cW {p : Ptr} {l : Nat} (h : inB wbs p l = true) : Covers [⟨pa s p, theorem Lay.post {s' : State} {W : List Region} (hP : PostB s s' W) : Lay rbs wbs s' := by have e : ∀ b ∈ rbs ++ wbs, s'.gpr b.1 = s.gpr b.1 := fun b hb => hP.bs _ (L.bs b hb) refine ⟨L.small, fun b hb b' hb' hne hw => ?_, fun b hb => ?_, fun b hb => ?_, fun b hb => ?_, fun b hb => ?_, - fun b hb => ?_, L.bs, by rw [hP.rsp]; exact L.sp24⟩ + fun b hb => ?_, L.bs, by rw [hP.rsp]; exact L.sp32⟩ · rw [e b hb, e b' hb']; exact L.dj b hb b' hb' hne hw · rw [e b hb, hP.rsp]; exact L.stk b hb · rw [e b hb]; exact L.nw b hb @@ -388,7 +388,7 @@ variable {rbs wbs : List (Reg × Nat)} {s s' : State} (L : Lay rbs wbs s) {ws : include L theorem Lay.fdisj (hc : keepB (rbs ++ wbs) ws p l = true) : - ∀ r ∈ ws.map (toR s) ++ [below (s.gpr .rsp) 24], Region.Disjoint ⟨pa s p, l⟩ r := by + ∀ r ∈ ws.map (toR s) ++ [below (s.gpr .rsp) 32], Region.Disjoint ⟨pa s p, l⟩ r := by simp only [keepB, Bool.and_eq_true, List.all_eq_true] at hc obtain ⟨⟨_, hin⟩, hall⟩ := hc intro r hr diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Blocks.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Blocks.lean index ceeabaf25..77da47651 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Blocks.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Blocks.lean @@ -119,18 +119,24 @@ theorem maskBody_ok (s : State) (h0 : InRegions (s.rd ++ s.wr) (s.gpr .rdi) 4) ( refine WP.keep _ ?_ (by decide) xrun [h0, h1] -/-- The mask of the polynomial at `a` by `eax`: each coefficient `∧ -eax`. -/ -theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a : Ptr} - (hi : inB (rbs ++ wbs) a 1024 = true) (hw : inB wbs a 1024 = true) : - WP isa (mask a) s fun s' => PPostB s s' [(a, 1024)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ - ∀ i < n, coeffAt s'.mem (pa s a) i = coeffAt s.mem (pa s a) i &&& (0 - (s.gpr .rax).setWidth 32) := by +theorem coeffAt_writeW' (m : Mem) (p : Addr) {N i j : Nat} (hN : 4 * N ≤ 2 ^ 64) (hi : i < N) (hj : j < N) + (v : BitVec 32) : + coeffAt (m.writeW (p + BitVec.ofNat 64 (4 * j)) v) p i = if j = i then v else coeffAt m p i := by + split + · subst j; exact Mem.readW_writeW_self32 m _ v + · exact Mem.readW_writeW_sep (Offset.sep p (by omega) (by omega) (by omega)) (by decide) + +/-- The mask of the `N` coefficients from `a` by `eax`: each `∧ -eax`. -/ +theorem maskN_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a : Ptr} {N : Nat} (hN0 : 0 < N) + (hN : N < 2 ^ 29) (hi : inB (rbs ++ wbs) a (4 * N) = true) (hw : inB wbs a (4 * N) = true) : + WP isa (mask a N) s fun s' => PPostB s s' [(a, 4 * N)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ + ∀ i < N, coeffAt s'.mem (pa s a) i = coeffAt s.mem (pa s a) i &&& (0 - (s.gpr .rax).setWidth 32) := by have hS := L.ok - have hok : ∀ x ∈ ([(.rdi, .ptr a), (.rcx, .imm 256)] : List (Reg × Arg)), x.2.Ok ∧ x.1 ∈ argRegs := by + have hok : ∀ x ∈ ([(.rdi, .ptr a), (.rcx, .imm N)] : List (Reg × Arg)), x.2.Ok ∧ x.1 ∈ argRegs := by simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] - exact ⟨⟨ptr_ok hS hi, by decide⟩, ⟨show 256 < 2 ^ 31 by decide, by decide⟩⟩ + exact ⟨⟨ptr_ok hS hi, by decide⟩, ⟨show N < 2 ^ 31 by omega, by decide⟩⟩ have hrd := L.inR hi have hwr := L.inW hw - have nn : n = 256 := rfl unfold mask refine WP.seq ?_ rw [WP.block_append_iff] @@ -142,14 +148,14 @@ theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a simp only [List.mem_singleton]; intro h; rw [h] at hb; exact absurd hb (by decide) have e1 : s1.gpr .rdi = pa s a := by rw [hv1 _ (List.mem_cons_self ..)]; simp only [Arg.val, pa]; rw [k₀.gpr nb] - have e2 : s1.gpr .rcx = BitVec.ofNat 64 256 := hv1 _ (List.mem_cons_of_mem _ (List.mem_cons_self ..)) + have e2 : s1.gpr .rcx = BitVec.ofNat 64 N := hv1 _ (List.mem_cons_of_mem _ (List.mem_cons_self ..)) have hd1 : (s1.gpr .rdx).setWidth 32 = 0 - (s.gpr .rax).setWidth 32 := by rw [k1.gpr (by simp), hd₀] have k01 : Keep [.rdx, .rdi, .rcx] s s1 := (k₀.trans k1).mono (by simp) have hm01 : s1.mem = s.mem := hm1.trans hm₀ - refine WP.mono (wp_countdown (cnt := .rcx) (N := 256) (by decide) (by decide) (fun k s' => + refine WP.mono (wp_countdown (cnt := .rcx) (N := N) (by omega) hN0 (fun k s' => s'.gpr .rdi = pa s a + BitVec.ofNat 64 (4 * k) ∧ (s'.gpr .rdx).setWidth 32 = 0 - (s.gpr .rax).setWidth 32 ∧ - Frame [⟨pa s a, 1024⟩] s.mem s'.mem ∧ - (∀ i < n, coeffAt s'.mem (pa s a) i = + Frame [⟨pa s a, 4 * N⟩] s.mem s'.mem ∧ + (∀ i < N, coeffAt s'.mem (pa s a) i = if i < k then coeffAt s.mem (pa s a) i &&& (0 - (s.gpr .rax).setWidth 32) else coeffAt s.mem (pa s a) i) ∧ Keep [.rdx, .rdi, .rcx, .rax] s s') (fun k hk s' ⟨hdi, hdx, hf, hc, kk⟩ _ => ?_) (fun _ h => h) @@ -169,7 +175,7 @@ theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a have := hc k (by omega) rw [ifn (Nat.lt_irrefl _)] at this exact this - rw [hm, hdi, hck, hdx, Proof.MlDsa.Verify.coeffAt_writeW _ _ hi (by omega)] + rw [hm, hdi, hck, hdx, coeffAt_writeW' _ _ (N := N) (by omega) hi (by omega)] by_cases e : k = i · subst e; rw [ifp rfl, ifp (Nat.lt_succ_self _)] · rw [ifn e, hc i hi] @@ -177,4 +183,11 @@ theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a · rw [ifp h', ifp (by omega)] · rw [ifn h', ifn (by omega)] +/-- The mask of the polynomial at `a` by `eax`: each coefficient `∧ -eax`. -/ +theorem mask_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a : Ptr} + (hi : inB (rbs ++ wbs) a 1024 = true) (hw : inB wbs a 1024 = true) : + WP isa (mask a) s fun s' => PPostB s s' [(a, 1024)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ + ∀ i < n, coeffAt s'.mem (pa s a) i = coeffAt s.mem (pa s a) i &&& (0 - (s.gpr .rax).setWidth 32) := + maskN_ok L (N := 256) (by decide) (by decide) hi hw + end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTBase.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTBase.lean index 1168a716f..8d1c9c257 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTBase.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTBase.lean @@ -67,13 +67,13 @@ theorem copy_tr {dst src : Ptr} {n : Nat} (hok : ∀ a ∈ copyArgs dst src n, a · rw [hx.1.1 _ (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..))), hy.1.1 _ (List.mem_cons_of_mem _ (List.mem_cons_of_mem _ (List.mem_cons_self ..)))]; rfl -theorem maskPre_wp {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) (s : State) : +theorem maskPre_wp {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) {N : Nat} (hN : N < 2 ^ 31) (s : State) : WP isa (.block (([.mov32 .rdx (.imm 0), .alu32 .sub .rdx (.reg .rax)] : List Instr) ++ - glue [(.rdi, .ptr a), (.rcx, .imm 256)])) s - fun s' => s'.gpr .rdi = (Arg.ptr a).val s ∧ s'.gpr .rcx = BitVec.ofNat 64 256 := by - have hok' : ∀ x ∈ ([(.rdi, .ptr a), (.rcx, .imm 256)] : List (Reg × Arg)), x.2.Ok ∧ x.1 ∈ argRegs := by + glue [(.rdi, .ptr a), (.rcx, .imm N)])) s + fun s' => s'.gpr .rdi = (Arg.ptr a).val s ∧ s'.gpr .rcx = BitVec.ofNat 64 N := by + have hok' : ∀ x ∈ ([(.rdi, .ptr a), (.rcx, .imm N)] : List (Reg × Arg)), x.2.Ok ∧ x.1 ∈ argRegs := by simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] - exact ⟨⟨hok, by decide⟩, ⟨show 256 < 2 ^ 31 by decide, by decide⟩⟩ + exact ⟨⟨hok, by decide⟩, ⟨hN, by decide⟩⟩ rw [WP.block_append_iff] refine WP.mono (maskPre_ok s) fun s₀ ⟨_, k₀⟩ => ?_ refine WP.mono (glue_ok _ hok' (by simp only [List.map_cons, List.map_nil]; decide) s₀) fun s1 ⟨⟨hv1, _⟩, _⟩ => @@ -84,10 +84,11 @@ theorem maskPre_wp {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) (s : St simp only [Arg.val, pa] rw [k₀.gpr nb] -theorem mask_tr {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) {P : State → State → Prop} - (hP : ∀ x y, P x y → SameB x y) : RelCT isa P (mask a) fun _ _ => True := by +theorem mask_tr {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) {N : Nat} (hN : N < 2 ^ 31) + {P : State → State → Prop} (hP : ∀ x y, P x y → SameB x y) : RelCT isa P (mask a N) fun _ _ => True := by unfold mask - refine blockLoop_tr (fun i hi s => ?_) (fun x y _ => ⟨maskPre_wp hok hb x, maskPre_wp hok hb y⟩) [.rdi, .rcx] + refine blockLoop_tr (fun i hi s => ?_) (fun x y _ => ⟨maskPre_wp hok hb hN x, maskPre_wp hok hb hN y⟩) + [.rdi, .rcx] (fun x y x' y' hp hx hy r hr => ?_) (by taint_decide) · rcases List.mem_append.mp hi with h | h · simp only [List.mem_cons, List.not_mem_nil, or_false] at h; rcases h with rfl | rfl <;> rfl diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean index 3c5c9a2c1..de99c2d5f 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean @@ -40,7 +40,7 @@ theorem aOne_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r have hkl := kl_le p hp simp only [aChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, Bool.or_eq_true, Bool.not_eq_true', decide_eq_false_iff_not, Nat.not_lt] at hck - obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hck + obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hck have hT : ∀ σ s, IA p r c σ s → T p σ s := fun _ _ h => h.t refine relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (aOne_ok C hp hv hr hc hs) fun _ h' => ⟨h, hh, h'⟩) ?_ have em : (8 * r + c) % 8 = c := by omega @@ -64,17 +64,94 @@ theorem aOne_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r exact ⟨WP.mono (rejNttAt_ok C.rejNtt L.1 hrej) fun _ h' => ⟨_, h'.1⟩, WP.mono (rejNttAt_ok C.rejNtt L.2.1 hrej) fun _ h' => ⟨_, h'.1⟩⟩ +theorem bytes136 (m : Mem) (P : Addr) : bytesAt m P 136 = bytesAt m P 34 ++ bytesAt m (P + BitVec.ofNat 64 34) 34 ++ + bytesAt m (P + BitVec.ofNat 64 68) 34 ++ bytesAt m (P + BitVec.ofNat 64 102) 34 := by + rw [show 136 = 34 + 102 from rfl, Proof.MlKem.bytesAt_add, show 102 = 34 + 68 from rfl, Proof.MlKem.bytesAt_add, + show 68 = 34 + 34 from rfl, Proof.MlKem.bytesAt_add] + simp only [BitVec.add_assoc, ← BitVec.ofNat_add, List.append_assoc, Nat.reduceAdd] + +/-- The four seeds of `SB4`, for the entries `(r, c₀), …, (r, c₀ + 3)`. -/ +theorem GS.seeds {p : Params} {h : List (Vector Bool n)} {r c c₀ : Nat} {σ s : State} (g : GS p h r c c₀ 4 σ s) : + bytesAt s.mem (pa s (sc oSB4)) 136 = aSeed (vPk p σ) r (c₀ + 0) ++ aSeed (vPk p σ) r (c₀ + 1) ++ + aSeed (vPk p σ) r (c₀ + 2) ++ aSeed (vPk p σ) r (c₀ + 3) := by + have e : ∀ k, pa s (sc (oSB4 + 34 * k)) = pa s (sc oSB4) + BitVec.ofNat 64 (34 * k) := fun k => by + simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add] + have b0 : bytesAt s.mem (pa s (sc oSB4)) 34 = aSeed (vPk p σ) r (c₀ + 0) := g.done 0 (by decide) + have b1 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 34) 34 = aSeed (vPk p σ) r (c₀ + 1) := by + have := g.done 1 (by decide); rw [e] at this; exact this + have b2 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 68) 34 = aSeed (vPk p σ) r (c₀ + 2) := by + have := g.done 2 (by decide); rw [e] at this; exact this + have b3 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 102) 34 = aSeed (vPk p σ) r (c₀ + 3) := by + have := g.done 3 (by decide); rw [e] at this; exact this + rw [bytes136, b0, b1, b2, b3] + +theorem slotBlock_tr {r c₀ j : Nat} {P : State → State → Prop} (hP : ∀ s₁ s₂, P s₁ s₂ → s₁.gpr .rbx = s₂.gpr .rbx) : + RelCT isa P (.block (setSR r c₀ j)) fun _ _ => True := + taintRel [.rbx] (fun s₁ s₂ h r hr => by simp only [List.mem_singleton] at hr; subst hr; exact hP s₁ s₂ h) + (hc := .block []) (by with_unfolding_all rfl) + +/-- The bytes of seed `j` of `SB4`, from `GS j` to `GS (j + 1)`. -/ +theorem slot_tr {p : Params} (hp : p ∈ params) {r c c₀ j : Nat} (hr : r < 256) (hj : j < 4) (hx : c₀ + j < 256) + (hck : slotChk p r c j = true) : + RelCT isa (RV p fun σ s => ∃ h, HN p σ h ∧ GS p h r c c₀ j σ s) (.block (setSR r c₀ j)) + (RV p fun σ s => ∃ h, HN p σ h ∧ GS p h r c c₀ (j + 1) σ s) := + relInv (fun σ s hv ⟨h, hh, g⟩ => WP.mono (slot_ok hp hv hr hj hx hck g) fun _ g' => ⟨h, hh, g'⟩) + (slotBlock_tr fun x y h => (RV.lrel hp (fun _ _ h => let ⟨_, _, g⟩ := h; g.s3.t) h).2.2.1 .rbx (by decide)) + +theorem aGrp_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r c₀ c : Nat} (hr : r < p.k) + (hck : gChk p r c₀ c = true) : RelCT isa (RV p (IA p r c)) (aGrp P p r c₀) (RV p (IA p r (c₀ + 4))) := by + have hkl := kl_le p hp + have hck' := hck + simp only [gChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, decide_eq_true_eq] at hck' + obtain ⟨⟨⟨⟨⟨⟨⟨hsl, hrej⟩, hin⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hck' + have hT : ∀ σ s, IA p r c σ s → T p σ s := fun _ _ h => h.t + have hTG : ∀ σ s, (∃ h, HN p σ h ∧ GS p h r c c₀ 4 σ s) → T p σ s := fun _ _ ⟨_, _, g⟩ => g.s3.t + refine relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (aGrp_ok C hp hv hr hck hs) fun _ h' => ⟨h, hh, h'⟩) ?_ + unfold aGrp + refine RelCT.seq (RelCT.mono (slot_tr hp (by omega) (by decide) (by omega) (hsl 0 (by decide))) + (fun x y ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁⟩, ⟨h₂, hh₂, s₂⟩⟩ => ⟨σ₁, σ₂, v₁, v₂, pub, + ⟨h₁, hh₁, s₁, fun _ h => absurd h (by omega)⟩, ⟨h₂, hh₂, s₂, fun _ h => absurd h (by omega)⟩⟩) + fun _ _ h => h) ?_ + refine RelCT.seq (slot_tr hp (by omega) (by decide) (by omega) (hsl 1 (by decide))) ?_ + refine RelCT.seq (slot_tr hp (by omega) (by decide) (by omega) (hsl 2 (by decide))) ?_ + refine RelCT.seq (slot_tr hp (by omega) (by decide) (by omega) (hsl 3 (by decide))) ?_ + unfold sampled4 + refine RelCT.seq (RelCT.sameB (rej4At_tr C.rej4 (layOk p hp) hrej fun x y h => ?_) (fun x y h => ?_) + (fun x y h => (RV.lrel hp hTG h).2.2)) + (sampledTail4_tr (ptr_ok (layOk p hp) hin) (show Reg.rbx ∈ bases by decide)) + · have L := RV.lrel hp hTG h + obtain ⟨σ₁, σ₂, _, _, pub, ⟨_, _, gx⟩, ⟨_, _, gy⟩⟩ := h + exact ⟨L.1, L.2.1, L.2.2, by rw [gx.seeds, gy.seeds, pub.2.2.2.2.2.1]⟩ + · have L := RV.lrel hp hTG h + exact ⟨WP.mono (rej4At_ok C.rej4 L.1 hrej) fun _ h' => ⟨_, h'.1⟩, + WP.mono (rej4At_ok C.rej4 L.2.1 hrej) fun _ h' => ⟨_, h'.1⟩⟩ + +theorem IA.next {p : Params} {r : Nat} {x y : State} (h : RV p (IA p r p.ℓ) x y) : RV p (IA p (r + 1) 0) x y := by + obtain ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁⟩, ⟨h₂, hh₂, s₂⟩⟩ := h + exact ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁.next⟩, ⟨h₂, hh₂, s₂.next⟩⟩ + theorem aRow_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r : Nat} (hr : r < p.k) : RelCT isa (RV p (IA p r 0)) (aRow P p r) (RV p (IA p (r + 1) 0)) := by + have hkl := kl_le p hp + have hg := gChk_all p hp r hr + have hl : p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by + have : ∀ p ∈ params, p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by decide + exact this p hp unfold aRow - refine RelCT.mono (seqR_tr (R := fun e => RV p (IA p r (e - 8 * r))) p.ℓ (8 * r) fun e he he' => ?_) - (fun x y h => by rwa [Nat.sub_self]) fun x y h => ?_ - · have := aOne_tr C hp hr (c := e - 8 * r) (by omega) - rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this - exact this - · rw [show 8 * r + p.ℓ - 8 * r = p.ℓ by omega] at h - obtain ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁⟩, ⟨h₂, hh₂, s₂⟩⟩ := h - exact ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁.next⟩, ⟨h₂, hh₂, s₂.next⟩⟩ + refine RelCT.seq (aGrp_tr C hp hr hg.1) ?_ + by_cases h7 : p.ℓ = 7 + · rw [ite_eq_left h7] + refine RelCT.mono (aGrp_tr C hp hr (hg.2 h7)) (fun x y h => by rwa [Nat.zero_add] at h) fun x y h => ?_ + rw [show 3 + 4 = p.ℓ by omega] at h + exact IA.next h + · rw [ite_eq_right h7] + refine RelCT.mono (seqR_tr (R := fun e => RV p (IA p r (e - 8 * r))) (p.ℓ - 4) (8 * r + 4) fun e he he' => ?_) + (fun x y h => by rw [show 8 * r + 4 - 8 * r = 4 by omega]; rwa [Nat.zero_add] at h) fun x y h => ?_ + · have := aOne_tr C hp hr (c := e - 8 * r) (by omega) + rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this + exact this + · rw [show 8 * r + 4 + (p.ℓ - 4) - 8 * r = p.ℓ by omega] at h + exact IA.next h theorem ballStage_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) : RelCT isa (RV p (IA p p.k 0)) (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC) (RV p (I4 p)) := by @@ -94,6 +171,22 @@ theorem ballStage_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params exact ⟨WP.mono (ballAt_ok C.ball L.1 c6 c5) fun _ h' => ⟨_, h'.1⟩, WP.mono (ballAt_ok C.ball L.2.1 c6 c5) fun _ h' => ⟨_, h'.1⟩⟩ +/-- While copying `ρ`. -/ +def IRho (p : Params) (j : Nat) (σ s : State) : Prop := ∃ h, HN p σ h ∧ RhoS p h j σ s + +theorem copyK_tr {p : Params} (hp : p ∈ params) {j : Nat} (hj : j < 4) : + RelCT isa (RV p (IRho p j)) (copy (sc (oSB4 + 34 * j)) (.rbp, 0) 32) (RV p (IRho p (j + 1))) := by + have hc := rChk_all p hp j hj + simp only [rChk, Bool.and_eq_true] at hc + have hS := layOk p hp + have hsd' := sepB_spec hc.1.1.1.1 + have hok : ∀ a ∈ copyArgs (sc (oSB4 + 34 * j)) (.rbp, 0) 32, a.2.Ok ∧ a.1 ∈ argRegs := by + simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] + exact ⟨⟨ptr_ok hS hsd'.2.1, by decide⟩, ⟨ptr_ok hS hsd'.1, by decide⟩, ⟨show 32 < 2 ^ 31 by decide, by decide⟩⟩ + exact relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (copyK_ok hp hv hj hs) fun _ h' => ⟨h, hh, h'⟩) + (copy_tr hok (show Reg.rbx ∈ bases by decide) (by decide) fun x y h => + (RV.lrel hp (fun _ _ h => let ⟨_, _, hs⟩ := h; hs.t) h).2.2) + theorem samples_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) : RelCT isa (RV p (Is p)) (samples P p) (RV p (I4 p)) := by have hc := sChk_all p hp @@ -104,11 +197,16 @@ theorem samples_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) have hok : ∀ a ∈ copyArgs (sc oSB) (.rbp, 0) 32, a.2.Ok ∧ a.1 ∈ argRegs := by simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] exact ⟨⟨ptr_ok hS hsd'.2.1, by decide⟩, ⟨ptr_ok hS hsd'.1, by decide⟩, ⟨show 32 < 2 ^ 31 by decide, by decide⟩⟩ - unfold samples - refine RelCT.seq (relInv (I' := IA p 0 0) + unfold samples rhos + refine RelCT.seq (RelCT.seq (relInv (I' := IRho p 0) (fun σ s hv ⟨h, hh, hs, h15⟩ => WP.mono (copyRho_ok hp hv hs h15) fun _ h' => ⟨h, hh, h'⟩) (copy_tr hok (by decide) (by decide) fun x y h => - (RV.lrel hp (fun _ _ h => let ⟨_, _, hs, _⟩ := h; hs.t) h).2.2)) (RelCT.seq ?_ (ballStage_tr C hp)) + (RV.lrel hp (fun _ _ h => let ⟨_, _, hs, _⟩ := h; hs.t) h).2.2)) + (RelCT.seq (copyK_tr hp (j := 0) (by decide)) (RelCT.seq (copyK_tr hp (j := 1) (by decide)) + (RelCT.seq (copyK_tr hp (j := 2) (by decide)) (RelCT.mono (copyK_tr hp (j := 3) (by decide)) (fun _ _ h => h) + (Q' := RV p (IA p 0 0)) fun x y h => ?_))))) (RelCT.seq ?_ (ballStage_tr C hp)) + · obtain ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, r₁⟩, ⟨h₂, hh₂, r₂⟩⟩ := h + exact ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, r₁.s3⟩, ⟨h₂, hh₂, r₂.s3⟩⟩ have := seqR_tr (R := fun r => RV p (IA p r 0)) p.k 0 fun r _ hr => aRow_tr C hp (by omega) rwa [Nat.zero_add] at this diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTStages.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTStages.lean index 9933336e8..9647e78c8 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTStages.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTStages.lean @@ -43,7 +43,12 @@ theorem and15_post (x : State) : WP isa (.block and15) x fun x' => ∃ W, PostB theorem sampledTail_tr {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) : RelCT isa (fun x y => SameB x y) (.seq (.block and15) (mask a)) fun _ _ => True := RelCT.seq (RelCT.sameB and15_tr (fun x y _ => ⟨and15_post x, and15_post y⟩) fun _ _ h => h) - (mask_tr hok hb fun _ _ h => h) + (mask_tr hok hb (N := 256) (by decide) fun _ _ h => h) + +theorem sampledTail4_tr {a : Ptr} (hok : (Arg.ptr a).Ok) (hb : a.1 ∈ bases) : + RelCT isa (fun x y => SameB x y) (.seq (.block and15) (mask a 1024)) fun _ _ => True := + RelCT.seq (RelCT.sameB and15_tr (fun x y _ => ⟨and15_post x, and15_post y⟩) fun _ _ h => h) + (mask_tr hok hb (N := 1024) (by decide) fun _ _ h => h) theorem flag_ne {P : Prop} [Decidable P] (h : (flag P).setWidth 32 ≠ 0) : P := by by_contra hn diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Call.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Call.lean index e1e56ef2b..eb9c0af10 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Call.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Call.lean @@ -8,7 +8,7 @@ import VerifiedGarbage.Proof.Framework.Contract Untrusted: everything here is checked by Lean. A call of verified code, with the moves of its arguments before it (`callAt_ok`), leaves the permissions and the callee-saved registers as they were, and changes memory -only within the buffers it writes and the 24 bytes of stack below `rsp` +only within the buffers it writes and the 32 bytes of stack below `rsp` (`Post`); two runs whose arguments agree and whose callee's public data agree leak the same (`callAt_tr`). A callee may be verified against its contract with any stack up to 16 bytes: its precondition follows from the @@ -61,7 +61,7 @@ abbrev Args (as : List (Reg × Arg)) (s s1 : State) : Prop := theorem callAt_ok {n : String} {c : Prog isa} {k : Contract isa} (hv : ∀ s, k.pre s → ∃ t s', Exec isa c s t s' ∧ abiPreserved s s' ∧ k.post s s') - (hsp : NoSp c) (hd : c.depth ≤ 2) {as : List (Reg × Arg)} (hok : ∀ a ∈ as, a.2.Ok ∧ a.1 ∈ argRegs) + (hsp : NoSp c) (hd : c.depth ≤ 3) {as : List (Reg × Arg)} (hok : ∀ a ∈ as, a.2.Ok ∧ a.1 ∈ argRegs) (hnd : (as.map (·.1)).Nodup) {s : State} {rd wr : List Region} (hpre : ∀ s1, Args as s s1 → k.pre (s1.callEntry.withRegions rd wr)) (hc : Covers (rd ++ wr) (s.rd ++ s.wr)) (hw : Covers wr s.wr) : diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CallSample.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CallSample.lean index 67939bf56..7d71683f7 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CallSample.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CallSample.lean @@ -96,6 +96,96 @@ theorem rejNttAt_tr {P : Prims} (C : CalleeOk P.rejNtt (rejNTTContract X86_64.ab rw [Lx.wbytesAt c4, Ly.wbytesAt c4, eb] exact ⟨by rw [e.2], rfl, e.pa (ptr_bs hS c4), e.pa (ptr_bs hS c5), e.pa (ptr_bs hS c6)⟩ +/-! ## `RejNTTPoly` four times -/ + +def rej4Chk (bs wbs : List (Reg × Nat)) (a w : Ptr) : Bool := + sepB bs (sc oSB4) 136 a 4096 && sepB bs (sc oSB4) 136 w 8192 && sepB bs a 4096 w 8192 && + inB bs (sc oSB4) 136 && inB bs a 4096 && inB bs w 8192 && inB wbs a 4096 && inB wbs w 8192 + +abbrev rej4Args (a w : Ptr) : List (Reg × Arg) := [(.rdi, .ptr (sc oSB4)), (.rsi, .ptr a), (.rdx, .ptr w)] + +theorem rej4_args {bs wbs : List (Reg × Nat)} (L : LayOk bs) {a w : Ptr} (hc : rej4Chk bs wbs a w = true) : + ∀ x ∈ rej4Args a w, x.2.Ok ∧ x.1 ∈ argRegs := by + simp only [rej4Chk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨⟨⟨_, c4⟩, c5⟩, c6⟩, _⟩, _⟩ := hc + simp only [List.forall_mem_cons, List.not_mem_nil, false_implies, implies_true, and_true] + exact ⟨⟨ptr_ok L c4, by decide⟩, ⟨ptr_ok L c5, by decide⟩, ⟨ptr_ok L c6, by decide⟩⟩ + +section +variable {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a w : Ptr} + (hc : rej4Chk (rbs ++ wbs) wbs a w = true) +include L hc + +theorem rej4_cov : Covers ([⟨pa s (sc oSB4), 136⟩] ++ [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩]) (s.rd ++ s.wr) ∧ + Covers [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩] s.wr := by + simp only [rej4Chk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨⟨⟨_, c4⟩, _⟩, _⟩, c7⟩, c8⟩ := hc + exact ⟨covers_append (L.cR c4) (covers_wr (covers_cons (L.cW c7) (L.cW c8))), covers_cons (L.cW c7) (L.cW c8)⟩ + +theorem rej4_pre {s1 : State} (h1 : Args (rej4Args a w) s s1) : + (rejNTT4Contract X86_64.abi 24).pre + (s1.callEntry.withRegions [⟨pa s (sc oSB4), 136⟩] [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩]) := by + simp only [rej4Chk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨⟨⟨⟨⟨c1, c2⟩, c3⟩, c4⟩, c5⟩, c6⟩, _⟩, _⟩ := hc + have g1 : s1.gpr .rdi = pa s (sc oSB4) := h1.r0 + have g2 : s1.gpr .rsi = pa s a := h1.r1 + have g3 : s1.gpr .rdx = pa s w := h1.r2 + sig_pre [rejNTT4Contract, rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] + rw [g1, g2, g3, h1.rsp] + exact ⟨L.sp24, rfl, rfl, L.disj c1, L.disj c2, L.disj c3, L.ret8 c4, L.ret8 c5, L.ret8 c6, L.stk24 c4, + L.stk24 c5, L.stk24 c6, L.nwp c4, L.nwp c5, L.nwp c6⟩ + +end + +theorem Lay.wseed4 {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) + (h : inB (rbs ++ wbs) (sc oSB4) 136 = true) (v : BitVec 64) {k : Nat} (hk : k < 4) : + seed4 (s.mem.writeW (s.gpr .rsp - 8) v) (pa s (sc oSB4)) k = seed4 s.mem (pa s (sc oSB4)) k := by + unfold seed4 + refine Proof.MlKem.bytesAt_congr fun i hi => ?_ + rw [BitVec.add_assoc, ← BitVec.ofNat_add] + exact L.wbytes h v (34 * k + i) (by omega) + +theorem rej4At_ok {P : Prims} (C : CalleeOk P.rej4 (rejNTT4Contract X86_64.abi 24)) {rbs wbs : List (Reg × Nat)} + {s : State} (L : Lay rbs wbs s) {a w : Ptr} (hc : rej4Chk (rbs ++ wbs) wbs a w = true) : + WP isa (rej4At P a w) s fun s' => PPostB s s' [(a, 4096), (w, 8192)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ + (res s' = 1 → ∀ k < 4, Reduced s'.mem (poly4 (pa s a) k)) ∧ + ((res s' = 1 ∧ ∀ k < 4, ∃ b : Bounds, rejNTTPoly b.rejNTT (seed4 s.mem (pa s (sc oSB4)) k) = + some (polyAt s'.mem (poly4 (pa s a) k))) ∨ + (res s' = 0 ∧ ∃ k < 4, rejNTTPoly minBounds.rejNTT (seed4 s.mem (pa s (sc oSB4)) k) = none)) := by + refine WP.mono (callAt_ok C.correct C.nosp C.depth (rej4_args L.ok hc) + (by simp only [List.map_cons, List.map_nil]; decide) + (fun s1 h1 => rej4_pre L hc h1) (rej4_cov L hc).1 (rej4_cov L hc).2) + fun s' ⟨hP, s1, h1, s₂, hm, hg, hq⟩ => ⟨hP.b, hP.cs .r15 (by decide), ?_⟩ + simp only [rej4Chk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨⟨⟨_, c4⟩, _⟩, _⟩, _⟩, _⟩ := hc + sig_post [rejNTT4Contract, rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] at hq + rw [h1.r0, h1.r1, hm, h1.rsp, h1.1.2, hg _ (by decide)] at hq + simp only [Arg.val] at hq + obtain ⟨hr, ho⟩ := hq + refine ⟨hr, ?_⟩ + rcases ho with ⟨h1', hb⟩ | ⟨h0, k, hk, hn⟩ + · exact .inl ⟨h1', fun k hk => by rw [← L.wseed4 c4 _ hk]; exact hb k hk⟩ + · exact .inr ⟨h0, k, hk, by rw [← L.wseed4 c4 _ hk]; exact hn⟩ + +theorem rej4At_tr {P : Prims} (C : CalleeOk P.rej4 (rejNTT4Contract X86_64.abi 24)) {rbs wbs : List (Reg × Nat)} + (hS : LayOk (rbs ++ wbs)) {a w : Ptr} (hc : rej4Chk (rbs ++ wbs) wbs a w = true) {Q : State → State → Prop} + (hQ : ∀ x y, Q x y → Lay rbs wbs x ∧ Lay rbs wbs y ∧ SameB x y ∧ + bytesAt x.mem (pa x (sc oSB4)) 136 = bytesAt y.mem (pa y (sc oSB4)) 136) : + RelCT isa Q (rej4At P a w) fun _ _ => True := by + refine callAt_tr C.correct C.ct (rej4_args hS hc) (by simp only [List.map_cons, List.map_nil]; decide) ?_ + intro x y x1 y1 hp h1 h2 + obtain ⟨Lx, Ly, e, eb⟩ := hQ x y hp + refine ⟨_, _, _, _, rej4_pre Lx hc h1, rej4_pre Ly hc h2, ?_, (rej4_cov Lx hc).1, (rej4_cov Lx hc).2, + (rej4_cov Ly hc).1, (rej4_cov Ly hc).2, e.2⟩ + have hc' := hc + simp only [rej4Chk, Bool.and_eq_true] at hc' + obtain ⟨⟨⟨⟨⟨_, c4⟩, c5⟩, c6⟩, _⟩, _⟩ := hc' + sig_pub [rejNTT4Contract, rejNTT4Sig, X86_64.abi, VG.X86_64.argRegs] + rw [h1.r0, h1.r1, h1.r2, h2.r0, h2.r1, h2.r2, h1.1.2, h2.1.2, h1.rsp, h2.rsp] + simp only [Arg.val] + rw [Lx.wbytesAt c4, Ly.wbytesAt c4, eb] + exact ⟨by rw [e.2], rfl, e.pa (ptr_bs hS c4), e.pa (ptr_bs hS c5), e.pa (ptr_bs hS c6)⟩ + /-! ## `SampleInBall` -/ def ballChk (bs wbs : List (Reg × Nat)) (ct : Ptr) (len : Nat) (c : Ptr) : Bool := diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean index 430f0bb51..fde6d0c05 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Entry.lean @@ -11,8 +11,9 @@ needs on its entry, from the layout of the caller: that its buffers are apart from its return address and the 16 bytes of stack below it, and from each other, and that they read on entry as they did before the call (`Ent`). And what the callees must be (`CalleeOk`): correct and constant -time under their contracts with 16 bytes of stack, not writing the stack -pointer, calling at most two deep, and never loading MXCSR. +time under their contracts with 16 bytes of stack (24 for +`vg_mldsa_rej_ntt_poly4`), not writing the stack pointer, calling at most +three deep, and never loading MXCSR. -/ namespace VG.Proof.MlDsa.X86_64.Verify @@ -25,14 +26,14 @@ open VG.Spec.Sha3 (bytesAt) /-! ## Callees -/ /-- A callee: correct and constant time under the contract `k` (a shared -contract with 16 bytes of stack), not writing `rsp`, calling at most two -deep, loading MXCSR only to restore it (`ctlOk`), and never writing the -stack pointer (which its callers' artifacts check). -/ +contract), not writing `rsp`, calling at most three deep, loading MXCSR only +to restore it (`ctlOk`), and never writing the stack pointer (which its +callers' artifacts check). -/ structure CalleeOk (c : Prog isa) (k : Contract isa) : Prop where correct : ∀ s, k.pre s → ∃ t s', Exec isa c s t s' ∧ abiPreserved s s' ∧ k.post s s' ct : ConstantTime isa k.pre k.pub c nosp : NoSp c - depth : c.depth ≤ 2 + depth : c.depth ≤ 3 ctl : ctlOk c = true spSafe : c.all (fun i => !isa.writesSp i) = true @@ -41,7 +42,7 @@ theorem CalleeOk.of_verified {c : Prog isa} {sig : Sig} {pre : Curry (sig.words {post : sig.Post X86_64.abi.ptrBits} {wa : Bool} {leak : Option (Curry (sig.words X86_64.abi.ptrBits) (Mem → List Nat))} {n : Nat} (h : Verified X86_64.target c (sig.contract X86_64.abi pre post wa n leak)) (hn : n ≤ 16) - (hsp : NoSp c) (hd : c.depth ≤ 2) (hmx : ctlOk c = true) + (hsp : NoSp c) (hd : c.depth ≤ 3) (hmx : ctlOk c = true) (hss : c.all (fun i => !isa.writesSp i) = true) : CalleeOk c (sig.contract X86_64.abi pre post wa 16 leak) := ⟨fun s hs => h.1 s (pre_stack hn hs), @@ -59,7 +60,7 @@ variable {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) include L theorem Lay.sp16 : 16 ≤ (s.gpr .rsp - 8).toNat := by - have := L.sp24 + have := L.sp32 rw [BitVec.toNat_sub, show (8 : BitVec 64).toNat = 8 from rfl] omega @@ -68,8 +69,8 @@ theorem Lay.ret8 {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : refine (L.stkD h).sub_left ?_ intro x hx simp only [Region.Contains] at hx ⊢ - rw [show x - (s.gpr .rsp - BitVec.ofNat 64 24) = (x - (s.gpr .rsp - 8)) + 16 by bv_omega, BitVec.toNat_add] - have : (16 : BitVec 64).toNat = 16 := rfl + rw [show x - (s.gpr .rsp - BitVec.ofNat 64 32) = (x - (s.gpr .rsp - 8)) + 24 by bv_omega, BitVec.toNat_add] + have : (24 : BitVec 64).toNat = 24 := rfl omega theorem Lay.stk16 {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : @@ -77,7 +78,21 @@ theorem Lay.stk16 {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : refine (L.stkD h).sub_left ?_ intro x hx simp only [Region.Contains] at hx ⊢ - rw [show x - (s.gpr .rsp - BitVec.ofNat 64 24) = x - (s.gpr .rsp - 8 - 16) by bv_omega] + rw [show x - (s.gpr .rsp - BitVec.ofNat 64 32) = (x - (s.gpr .rsp - 8 - 16)) + 8 by bv_omega, BitVec.toNat_add] + have : (8 : BitVec 64).toNat = 8 := rfl + omega + +theorem Lay.sp24 : 24 ≤ (s.gpr .rsp - 8).toNat := by + have := L.sp32 + rw [BitVec.toNat_sub, show (8 : BitVec 64).toNat = 8 from rfl] + omega + +theorem Lay.stk24 {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) : + Region.Disjoint ⟨s.gpr .rsp - 8 - 24, 24⟩ ⟨pa s p, l⟩ := by + refine (L.stkD h).sub_left ?_ + intro x hx + simp only [Region.Contains] at hx ⊢ + rw [show x - (s.gpr .rsp - BitVec.ofNat 64 32) = x - (s.gpr .rsp - 8 - 24) by bv_omega] omega theorem Lay.wbytes {p : Ptr} {l : Nat} (h : inB (rbs ++ wbs) p l = true) (v : BitVec 64) : diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Flag.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Flag.lean index 10d54497f..fe946ecd7 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Flag.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Flag.lean @@ -112,4 +112,57 @@ theorem sampled_ok {call : Prog isa} {rbs wbs : List (Reg × Nat)} {s : State} ( · rw [h1] at h0; cases h0 · exact hn +theorem coeffAt_poly4 (m : Mem) (p : Addr) (k j : Nat) : coeffAt m (poly4 p k) j = coeffAt m p (256 * k + j) := by + unfold coeffAt poly4 + rw [BitVec.add_assoc, ← BitVec.ofNat_add, show 1024 * k + 4 * j = 4 * (256 * k + j) by omega] + +/-- `sampled_ok` for a call that samples the four polynomials from `a`, each +from `F k`. -/ +theorem sampled4_ok {call : Prog isa} {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) {a : Ptr} + (hi : inB (rbs ++ wbs) a 4096 = true) (hw : inB wbs a 4096 = true) {p : Prop} [Decidable p] + (h15 : s.gpr .r15 = flag p) {ws : List (Ptr × Nat)} (haw : (a, 4096) ∈ ws) + {F : Nat → Bounds → Option Poly} + (hcall : WP isa call s fun s' => PPostB s s' ws ∧ s'.gpr .r15 = s.gpr .r15 ∧ + (res s' = 1 → ∀ k < 4, Reduced s'.mem (poly4 (pa s a) k)) ∧ + ((res s' = 1 ∧ ∀ k < 4, ∃ b, F k b = some (polyAt s'.mem (poly4 (pa s a) k))) ∨ + (res s' = 0 ∧ ∃ k < 4, F k minBounds = none))) : + WP isa (sampled4 call a) s fun s' => PPostB s s' ws ∧ (∀ k < 4, Reduced s'.mem (poly4 (pa s a) k)) ∧ + ∃ r : BitVec 32, (r = 1 ∨ r = 0) ∧ s'.gpr .r15 = flag (p ∧ r = 1) ∧ + (r = 1 → ∀ k < 4, ∃ b, F k b = some (polyAt s'.mem (poly4 (pa s a) k))) ∧ + (r = 0 → ∃ k < 4, F k minBounds = none) := by + unfold sampled4 + refine WP.seq (WP.mono hcall fun s₁ ⟨hP₁, h15₁, hr₁, ho₁⟩ => ?_) + have L₁ := L.post hP₁ + have hab := ptr_bs L.ok hi + have e₁ : pa s₁ a = pa s a := hP₁.pa hab + have hr : res s₁ = 1 ∨ res s₁ = 0 := by rcases ho₁ with ⟨h, _⟩ | ⟨h, _⟩ <;> [exact .inl h; exact .inr h] + refine WP.seq (WP.mono (and15_ok s₁) fun s₂ ⟨⟨h15₂, hm₂⟩, k₂⟩ => ?_) + have hP₂ : PPostB s₁ s₂ [] := postB_of_keep k₂ (by decide) (by rw [hm₂]; exact Frame.refl _ _) + have L₂ := L₁.post hP₂ + have e₂ : pa s₂ a = pa s₁ a := hP₂.pa hab + have hax : (s₂.gpr .rax).setWidth 32 = res s₁ := by rw [k₂.gpr (by decide)] + refine WP.mono (maskN_ok L₂ (N := 1024) (by decide) (by decide) hi hw) fun s₃ ⟨hP₃, h15₃, hc₃⟩ => ?_ + have hsub : ∀ w ∈ [(a, 4 * 1024)], w ∈ ws := by simp only [List.mem_singleton, forall_eq]; exact haw + have hcs : ∀ w ∈ [(a, 4 * 1024)], w.1.1 ∈ bases := by simp only [List.mem_singleton, forall_eq]; exact hab + have hP₁₂ : PPostB s s₂ ws := PPostB.trans hP₁ hP₂ (fun _ h => absurd h List.not_mem_nil) (fun w hw => hw) + (fun _ h => absurd h List.not_mem_nil) + have hP : PPostB s s₃ ws := PPostB.trans hP₁₂ hP₃ hcs (fun w hw => hw) hsub + rw [e₂, e₁, hax, hm₂] at hc₃ + have hc : ∀ k < 4, ∀ j < n, coeffAt s₃.mem (poly4 (pa s a) k) j = + coeffAt s₁.mem (poly4 (pa s a) k) j &&& (0 - res s₁) := fun k hk j hj => by + rw [coeffAt_poly4, coeffAt_poly4, hc₃ _ (by simp only [n] at hj; omega)] + refine ⟨hP, fun k hk => ?_, res s₁, hr, by rw [h15₃, h15₂, h15₁, h15, and_flag hr], fun h1 k hk => ?_, + fun h0 => ?_⟩ + · rcases hr with h1 | h0 + · exact reduced_of_coeffAt (fun i hi => by rw [hc k hk i hi, mask_one _ h1 i hi]) (hr₁ h1 k hk) + · exact (Proof.MlDsa.Verify.polyIs_zero fun i hi => by rw [hc k hk i hi, mask_zero _ h0 i hi]).1 + · rcases ho₁ with ⟨_, hb⟩ | ⟨h0, _⟩ + · obtain ⟨b, hb⟩ := hb k hk + refine ⟨b, ?_⟩ + rw [hb, polyAt_eq_of_coeffAt fun i hi => by rw [hc k hk i hi, mask_one _ h1 i hi]] + · rw [h1] at h0; cases h0 + · rcases ho₁ with ⟨h1, _⟩ | ⟨_, hn⟩ + · rw [h1] at h0; cases h0 + · exact hn + end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Hash.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Hash.lean index dd764ad70..380012376 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Hash.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Hash.lean @@ -44,8 +44,8 @@ variable {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) include L theorem kChk_spec (h : kChk (rbs ++ wbs) wbs = true) : - Region.Disjoint ⟨pa s (sc 0), 200⟩ ⟨pa s (sc 200), 640⟩ ∧ (below (s.gpr .rsp) 24).Disjoint ⟨pa s (sc 0), 200⟩ ∧ - (below (s.gpr .rsp) 24).Disjoint ⟨pa s (sc 200), 640⟩ ∧ Covers [⟨pa s (sc 0), 200⟩] s.wr ∧ + Region.Disjoint ⟨pa s (sc 0), 200⟩ ⟨pa s (sc 200), 640⟩ ∧ (below (s.gpr .rsp) 32).Disjoint ⟨pa s (sc 0), 200⟩ ∧ + (below (s.gpr .rsp) 32).Disjoint ⟨pa s (sc 200), 640⟩ ∧ Covers [⟨pa s (sc 0), 200⟩] s.wr ∧ Covers [⟨pa s (sc 200), 640⟩] s.wr := by simp only [kChk, Bool.and_eq_true] at h obtain ⟨⟨⟨⟨c1, c2⟩, c3⟩, c4⟩, c5⟩ := h @@ -98,7 +98,7 @@ theorem kabs_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) (h refine WP.seq (WP.mono (glue_ok' ha (by simp only [List.map_cons, List.map_nil]; decide) s) fun s1 (h1 : Args (kabsArgs src len 136 pos) s s1) => ?_) have hsp : s1.gpr .rsp = s.gpr .rsp := h1.rsp - have kk : ∀ {R : Region}, (below (s.gpr .rsp) 24).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := + have kk : ∀ {R : Region}, (below (s.gpr .rsp) 32).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := fun h => by rw [hsp]; exact h.sub_left (below_sub (by omega) (by omega)) refine absorb_call ⟨h1.r0, h1.r1, h1.r2, h1.r3, h1.r4, h1.r5, by decide, hpos, by omega, d1, (L.disj p1), (L.disj p2), kk k1, kk (L.stkD p3), kk k2⟩ @@ -113,7 +113,7 @@ theorem kabs_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) (h fun msg hm hpo => ?_⟩ · have f1 : Frame ([⟨pa s (sc 0), 200⟩, ⟨pa s (sc 200), 640⟩] ++ [below (s.gpr .rsp) 16]) s.mem s'.mem := by rw [← h1.1.2, ← hsp]; exact hf - exact Frame.below_mono (a := 16) (b := 24) f1 (by omega) (by omega) + exact Frame.below_mono (a := 16) (b := 32) f1 (by omega) (by omega) · rw [← h1.1.2] at hm ⊢ exact hR msg hm hpo @@ -125,7 +125,7 @@ theorem postB_call {s s1 s' : State} {as : List (Reg × Arg)} (h1 : Args as s s1 (hf : Frame (W ++ [below (s.gpr .rsp) 16]) s.mem s'.mem) : PostB s s' W := by have hsp : s1.gpr .rsp = s.gpr .rsp := h1.rsp refine ⟨hrd.trans h1.2.2.1, hwr.trans h1.2.2.2, fun r hr => ?_, by rw [hcs _ (by decide), hsp], - Frame.below_mono (a := 16) (b := 24) hf (by omega) (by omega)⟩ + Frame.below_mono (a := 16) (b := 32) hf (by omega) (by omega)⟩ simp only [bases, List.mem_cons, List.not_mem_nil, or_false] at hr rw [hcs r (by rcases hr with rfl | rfl | rfl | rfl <;> decide), h1.2.gpr (by rcases hr with rfl | rfl | rfl | rfl <;> decide)] @@ -148,7 +148,7 @@ theorem kpad_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) (h refine WP.seq (WP.mono (glue_ok' ha (by simp only [List.map_cons, List.map_nil]; decide) s) fun s1 (h1 : Args (kpadArgs 136 pos 0x1f) s s1) => ?_) have hsp : s1.gpr .rsp = s.gpr .rsp := h1.rsp - have kk : ∀ {R : Region}, (below (s.gpr .rsp) 24).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := + have kk : ∀ {R : Region}, (below (s.gpr .rsp) 32).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := fun h => by rw [hsp]; exact h.sub_left (below_sub (by omega) (by omega)) refine pad_call ⟨h1.r0, h1.r1, h1.r2, h1.r4, by decide, hpos, d1, kk k1, kk k2⟩ (by rw [h1.2.2.1, h1.2.2.2]; exact covers_append covers_nil (covers_wr (covers_cons w1 w2))) @@ -185,7 +185,7 @@ theorem ksqz_ok {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay rbs wbs s) (h refine WP.seq (WP.mono (glue_ok' ha (by simp only [List.map_cons, List.map_nil]; decide) s) fun s1 (h1 : Args (ksqzArgs 136 out len) s s1) => ?_) have hsp : s1.gpr .rsp = s.gpr .rsp := h1.rsp - have kk : ∀ {R : Region}, (below (s.gpr .rsp) 24).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := + have kk : ∀ {R : Region}, (below (s.gpr .rsp) 32).Disjoint R → (below (s1.gpr .rsp) 16).Disjoint R := fun h => by rw [hsp]; exact h.sub_left (below_sub (by omega) (by omega)) refine squeeze_call ⟨h1.r0, h1.r1, h1.r2, h1.r3, h1.r4, h1.r5, by decide, by decide, by omega, (L.disj o1).symm, d1, L.disj o2, kk k1, kk (L.stkD o3), kk k2⟩ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/HashCT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/HashCT.lean index afbc83ecb..9d3c51ff0 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/HashCT.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/HashCT.lean @@ -51,7 +51,7 @@ theorem k_in {bs wbs : List (Reg × Nat)} (hk : kChk bs wbs = true) : inB bs (sc 0) 200 = true ∧ inB bs (sc 200) 640 = true := by simp only [kChk, Bool.and_eq_true] at hk; exact ⟨hk.1.1.1.2, hk.1.1.2⟩ -theorem kk16 {s s1 : State} (hsp : s1.gpr .rsp = s.gpr .rsp) {R : Region} (h : (below (s.gpr .rsp) 24).Disjoint R) : +theorem kk16 {s s1 : State} (hsp : s1.gpr .rsp = s.gpr .rsp) {R : Region} (h : (below (s.gpr .rsp) 32).Disjoint R) : (below (s1.gpr .rsp) 16).Disjoint R := by rw [hsp]; exact h.sub_left (below_sub (by omega) (by omega)) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean index 852e3e7a1..758076ee2 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean @@ -21,7 +21,7 @@ open VG.Spec.MlDsa /-- The primitives, each empty. -/ def P0 : Prims := ⟨.block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], - .block [], .block [], .block [], .block [], ""⟩ + .block [], .block [], .block [], .block [], .block [], ""⟩ /-- `q` holds of every instruction of the primitives `P`. -/ structure PrimsQ (q : Instr → Bool) (P : Prims) : Prop where @@ -38,6 +38,7 @@ structure PrimsQ (q : Instr → Bool) (P : Prims) : Prop where unpackT1 : P.unpackT1.allInstrs q = true hintUnpack : P.hintUnpack.allInstrs q = true normLt : P.normLt.allInstrs q = true + rej4 : P.rej4.allInstrs q = true /-- `q` holds of every instruction of `c` exactly when it does of `c'`. -/ def SameQ (q : Instr → Bool) (c c' : Prog isa) : Prop := c.allInstrs q = c'.allInstrs q @@ -68,6 +69,9 @@ theorem SameQ.ifOk {c c' : Prog isa} (h : SameQ q c c') : SameQ q (ifOk c) (ifOk theorem SameQ.sampled {c c' : Prog isa} (h : SameQ q c c') (a : Ptr) : SameQ q (sampled c a) (sampled c' a) := h.seq rfl +theorem SameQ.sampled4 {c c' : Prog isa} (h : SameQ q c c') (a : Ptr) : SameQ q (sampled4 c a) (sampled4 c' a) := + h.seq rfl + variable {P : Prims} (hP : PrimsQ q P) (p : Params) include hP @@ -80,7 +84,14 @@ omit p in theorem aOne_q (e : Nat) : SameQ q (aOne P e) (aOne P0 e) := SameQ.seq rfl (SameQ.sampled (SameQ.call hP.rejNtt _) _) -theorem aRow_q (r : Nat) : SameQ q (aRow P p r) (aRow P0 p r) := SameQ.seqR (aOne_q hP) _ _ +theorem aGrp_q (r c : Nat) : SameQ q (aGrp P p r c) (aGrp P0 p r c) := + SameQ.seq rfl (SameQ.seq rfl (SameQ.seq rfl (SameQ.seq rfl (SameQ.sampled4 (SameQ.call hP.rej4 _) _)))) + +theorem aRow_q (r : Nat) : SameQ q (aRow P p r) (aRow P0 p r) := by + refine (aGrp_q hP p r 0).seq ?_ + by_cases h : p.ℓ = 7 + · rw [ite_eq_left h, ite_eq_left h]; exact aGrp_q hP p r 3 + · rw [ite_eq_right h, ite_eq_right h]; exact SameQ.seqR (aOne_q hP) _ _ theorem samples_q : SameQ q (samples P p) (samples P0 p) := SameQ.seq rfl ((SameQ.seqR (aRow_q hP p) _ _).seq (SameQ.sampled (SameQ.call hP.ball _) _)) @@ -123,6 +134,7 @@ structure PrimsC (P : Prims) : Prop where unpackT1 : ctlOk P.unpackT1 = true hintUnpack : ctlOk P.hintUnpack = true normLt : ctlOk P.normLt = true + rej4 : ctlOk P.rej4 = true /-- `ctlC` holds of `c` exactly when it does of `c'`. -/ def SameC (c c' : Prog isa) : Prop := ctlC c = ctlC c' @@ -148,6 +160,9 @@ theorem SameC.ifOk {c c' : Prog isa} (h : SameC c c') : SameC (ifOk c) (ifOk c') theorem SameC.sampled {c c' : Prog isa} (h : SameC c c') (a : Ptr) : SameC (sampled c a) (sampled c' a) := h.seq rfl +theorem SameC.sampled4 {c c' : Prog isa} (h : SameC c c') (a : Ptr) : SameC (sampled4 c a) (sampled4 c' a) := + h.seq rfl + section variable {P : Prims} (hP : PrimsC P) (p : Params) include hP @@ -161,8 +176,15 @@ theorem verify_c : SameC (verify P p) (verify P0 p) := by (dot r).seq ((SameC.call hP.unpackT1 _).seq ((SameC.call hP.ntt _).seq ((SameC.call hP.mul _).seq ((SameC.call hP.sub _).seq ((SameC.call hP.invNtt _).seq ((SameC.call hP.useHint _).seq (SameC.call hP.simpleBitPack _))))))) + have aGrp : ∀ r c, SameC (aGrp P p r c) (aGrp P0 p r c) := fun r c => + SameC.seq rfl (SameC.seq rfl (SameC.seq rfl (SameC.seq rfl (SameC.sampled4 (SameC.call hP.rej4 _) _)))) + have aRow : ∀ r, SameC (aRow P p r) (aRow P0 p r) := fun r => by + refine (aGrp r 0).seq ?_ + by_cases h : p.ℓ = 7 + · rw [ite_eq_left h, ite_eq_left h]; exact aGrp r 3 + · rw [ite_eq_right h, ite_eq_right h]; exact SameC.seqR aOne _ _ have samples : SameC (samples P p) (samples P0 p) := - SameC.seq rfl ((SameC.seqR (fun r => SameC.seqR aOne _ _) _ _).seq + SameC.seq rfl ((SameC.seqR aRow _ _).seq (SameC.sampled (SameC.call hP.ball _) _)) have compute : SameC (compute P p) (compute P0 p) := (SameC.seqR (fun _ => SameC.call hP.ntt _) _ _).seq ((SameC.call hP.ntt _).seq @@ -183,7 +205,7 @@ include C hp theorem verify_ctl : ctlOk (verify P p) = true := ctlOk_of_ctlC ((verify_c ⟨C.ntt.ctl, C.invNtt.ctl, C.mul.ctl, C.mulAdd.ctl, C.sub.ctl, C.rejNtt.ctl, C.ball.ctl, C.useHint.ctl, C.simpleBitPack.ctl, C.bitUnpack.ctl, C.unpackT1.ctl, C.hintUnpack.ctl, - C.normLt.ctl⟩ p).trans (verify0_ctlC p hp)) + C.normLt.ctl, C.rej4.ctl⟩ p).trans (verify0_ctlC p hp)) theorem verify_spSafe : (verify P p).all (fun i => !isa.writesSp i) = true := Code.all_of_allInstrs ((verify_q ⟨Code.allInstrs_of_all C.ntt.spSafe, Code.allInstrs_of_all C.invNtt.spSafe, @@ -191,6 +213,7 @@ theorem verify_spSafe : (verify P p).all (fun i => !isa.writesSp i) = true := Code.allInstrs_of_all C.rejNtt.spSafe, Code.allInstrs_of_all C.ball.spSafe, Code.allInstrs_of_all C.useHint.spSafe, Code.allInstrs_of_all C.simpleBitPack.spSafe, Code.allInstrs_of_all C.bitUnpack.spSafe, Code.allInstrs_of_all C.unpackT1.spSafe, - Code.allInstrs_of_all C.hintUnpack.spSafe, Code.allInstrs_of_all C.normLt.spSafe⟩ p).trans (verify0_sp p hp)) + Code.allInstrs_of_all C.hintUnpack.spSafe, Code.allInstrs_of_all C.normLt.spSafe, + Code.allInstrs_of_all C.rej4.spSafe⟩ p).trans (verify0_sp p hp)) end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean index 8d87a8c4d..ef0be7ae9 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean @@ -44,6 +44,7 @@ def prims : Prims where unpackT1 := Pack.unpackT1 hintUnpack := Pack.hintBitUnpack normLt := Round.normLt + rej4 := Sample.Rej4.rejNTT4 /-- The primitives, with the polynomial arithmetic of `B`. -/ def primsWith (B : Arith.Backend) : Prims := @@ -53,6 +54,7 @@ def primsWith (B : Arith.Backend) : Prims := mul := B.mul mulAdd := B.mulAdd sub := B.sub + rej4 := B.rej4 sfx := B.sfx } /-- A function of the polynomial arithmetic satisfies what the proofs of verification need of it. -/ @@ -60,7 +62,7 @@ theorem calleeOf {sig : Sig} {pre : Curry (sig.words X86_64.abi.ptrBits) (Mem {post : sig.Post X86_64.abi.ptrBits} {wa : Bool} {c : Prog isa} (h : FnOk (fun S => sig.contract X86_64.abi pre post wa S none) c) : CalleeOk c (sig.contract X86_64.abi pre post wa 16 none) := - CalleeOk.of_verified h.ver (by decide) h.nosp h.depth h.ctl h.sp + CalleeOk.of_verified h.ver (by decide) h.nosp (Nat.le_succ_of_le h.depth) h.ctl h.sp theorem prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) where ntt := by @@ -106,11 +108,12 @@ theorem prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) where (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) (Code.all_of_allInstrs (by lit_decide)) : CalleeOk prims.normLt _) + rej4 := ⟨v.ok.rej4.ver.1, v.ok.rej4.ver.2.1, v.ok.rej4.nosp, v.ok.rej4.depth, v.ok.rej4.ctl, v.ok.rej4.sp⟩ /-- `vg_mldsa*_verify` for the parameter set `p`, calling the x86-64 primitives, with the polynomial arithmetic of `v`. -/ theorem verify_prims (v : ArithImpl) {p : Spec.MlDsa.Params} (hp : p ∈ params) : - Verified X86_64.target (verify (primsWith v.code) p) (Spec.MlDsa.verifyContract p X86_64.abi 24) := + Verified X86_64.target (verify (primsWith v.code) p) (Spec.MlDsa.verifyContract p X86_64.abi 32) := verify_verified (prims_okWith v) hp end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/PrimsOk.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/PrimsOk.lean index 5ad8ebd2e..3d87fe8e3 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/PrimsOk.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/PrimsOk.lean @@ -6,9 +6,9 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Verify.Control Untrusted: everything here is checked by Lean. `PrimsOk P`: each primitive of `P` is correct and constant time under its shared contract with at most 16 bytes of stack (`CalleeOk`, from its `Verified` proof by -`CalleeOk.of_verified`), never writes the stack pointer, calls at most two -deep and never loads MXCSR. The proofs of `vg_mldsa*_verify` hold for any -such `P`. +`CalleeOk.of_verified`; 24 for `vg_mldsa_rej_ntt_poly4`), never writes the +stack pointer, calls at most three deep and never loads MXCSR. The proofs +of `vg_mldsa*_verify` hold for any such `P`. -/ namespace VG.Proof.MlDsa.X86_64.Verify @@ -31,5 +31,6 @@ structure PrimsOk (P : Prims) : Prop where unpackT1 : CalleeOk P.unpackT1 (unpackT1Contract X86_64.abi 16) hintUnpack : CalleeOk P.hintUnpack (hintBitUnpackContract X86_64.abi 16) normLt : CalleeOk P.normLt (normLtContract X86_64.abi 16) + rej4 : CalleeOk P.rej4 (rejNTT4Contract X86_64.abi 24) end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean index 79e5d8548..63bbf0e86 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean @@ -50,6 +50,7 @@ structure S3 (p : Params) (h : List (Vector Bool n)) (r c : Nat) (σ st : State) hint : HintIs st.mem (pa st (pH 0)) p.k h z : ∀ i < p.ℓ, PolyIs st.mem (pa st (pZ i)) (toRq (vZ p (vSig p σ) i)) rho : bytesAt st.mem (pa st (sc oSB)) 32 = vRho (vPk p σ) + rho4 : ∀ k < 4, bytesAt st.mem (pa st (sc (oSB4 + 34 * k))) 32 = vRho (vPk p σ) red : ∀ r' < p.k, ∀ c' < p.ℓ, Done r c r' c' → Reduced st.mem (pa st (pA r' c')) ok : ∃ q : Bool, st.gpr .r15 = flag (q = true) ∧ (q = true → ∀ r' < p.k, ∀ c' < p.ℓ, Done r c r' c' → @@ -69,7 +70,9 @@ def aChk (p : Params) (e : Nat) : Bool := inB (vB p) (pS (20 + e)) 1024 && inB (vW p) (pS (20 + e)) 1024 && keepChk p wsB && keepChk p (wsA e) && keepB (vB p) wsB (sc oSB) 32 && keepB (vB p) (wsA e) (sc oSB) 32 && (List.range p.k).all (fun r' => (List.range p.ℓ).all fun c' => !decide (8 * r' + c' < e) || - (keepB (vB p) wsB (pS (20 + 8 * r' + c')) 1024 && keepB (vB p) (wsA e) (pS (20 + 8 * r' + c')) 1024)) + (keepB (vB p) wsB (pS (20 + 8 * r' + c')) 1024 && keepB (vB p) (wsA e) (pS (20 + 8 * r' + c')) 1024)) && + (List.range 4).all (fun k => keepB (vB p) wsB (sc (oSB4 + 34 * k)) 32 && + keepB (vB p) (wsA e) (sc (oSB4 + 34 * k)) 32) theorem aChk_all : ∀ p ∈ params, ∀ r < p.k, ∀ c < p.ℓ, aChk p (8 * r + c) = true := by decide @@ -79,20 +82,22 @@ theorem keepChk_spec {p : Params} {ws : List (Ptr × Nat)} (h : keepChk p ws = t simp only [keepChk, Bool.and_eq_true, List.all_eq_true, List.mem_range] at h exact ⟨h.1.1, h.1.2, h.2⟩ -theorem setB2_ok {p : Params} {s : State} (L : Lay (vR p) (vW p) s) (h32 : inB (vW p) (sc (oSB + 32)) 1 = true) - (h33 : inB (vW p) (sc (oSB + 33)) 1 = true) (x y : Nat) (hx : x < 256) (hy : y < 256) : - WP isa (.block (setB (sc (oSB + 32)) x ++ setB (sc (oSB + 33)) y)) s fun s' => PPostB s s' wsB ∧ +theorem setB2At_ok {p : Params} {s : State} (L : Lay (vR p) (vW p) s) (B : Nat) + (h32 : inB (vW p) (sc (B + 32)) 1 = true) + (h33 : inB (vW p) (sc (B + 33)) 1 = true) (x y : Nat) (hx : x < 256) (hy : y < 256) : + WP isa (.block (setB (sc (B + 32)) x ++ setB (sc (B + 33)) y)) s fun s' => + PPostB s s' [(sc (B + 32), 1), (sc (B + 33), 1)] ∧ s'.gpr .r15 = s.gpr .r15 ∧ - s'.mem = (s.mem.writeW (pa s (sc oSB) + BitVec.ofNat 64 32) (BitVec.ofNat 8 x)).writeW - (pa s (sc oSB) + BitVec.ofNat 64 33) (BitVec.ofNat 8 y) := by - have e32 : pa s (sc (oSB + 32)) = pa s (sc oSB) + BitVec.ofNat 64 32 := by + s'.mem = (s.mem.writeW (pa s (sc B) + BitVec.ofNat 64 32) (BitVec.ofNat 8 x)).writeW + (pa s (sc B) + BitVec.ofNat 64 33) (BitVec.ofNat 8 y) := by + have e32 : pa s (sc (B + 32)) = pa s (sc B) + BitVec.ofNat 64 32 := by simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add] - have e33 : pa s (sc (oSB + 33)) = pa s (sc oSB) + BitVec.ofNat 64 33 := by + have e33 : pa s (sc (B + 33)) = pa s (sc B) + BitVec.ofNat 64 33 := by simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add] rw [WP.block_append_iff] - refine WP.mono (setB_ok _ x (by decide) hx s (L.inW h32)) fun s₁ ⟨hm₁, k₁⟩ => ?_ - have e₁ : pa s₁ (sc (oSB + 33)) = pa s (sc (oSB + 33)) := by simp only [pa]; rw [k₁.gpr (by decide)] - refine WP.mono (setB_ok _ y (by decide) hy s₁ (by rw [k₁.2.2, e₁]; exact L.inW h33)) fun s₂ ⟨hm₂, k₂⟩ => + refine WP.mono (setB_ok _ x (show Reg.rbx ≠ .rax by decide) hx s (L.inW h32)) fun s₁ ⟨hm₁, k₁⟩ => ?_ + have e₁ : pa s₁ (sc (B + 33)) = pa s (sc (B + 33)) := by simp only [pa]; rw [k₁.gpr (by decide)] + refine WP.mono (setB_ok _ y (show Reg.rbx ≠ .rax by decide) hy s₁ (by rw [k₁.2.2, e₁]; exact L.inW h33)) fun s₂ ⟨hm₂, k₂⟩ => ⟨postB_of_keep (k₁.trans k₂) (by decide) ?_, by rw [k₂.gpr (by decide), k₁.gpr (by decide)], ?_⟩ · rw [hm₂, hm₁, e₁] have hc : ∀ q : Ptr, (Region.mk (pa s q) 1).Contains (pa s q) 1 := fun q => Region.contains_self _ _ @@ -100,6 +105,13 @@ theorem setB2_ok {p : Params} {s : State} (L : Lay (vR p) (vW p) s) (h32 : inB ( (List.mem_cons_of_mem _ (List.mem_cons_self ..)) _ (hc _) · rw [hm₂, hm₁, e₁, e32, e33] +theorem setB2_ok {p : Params} {s : State} (L : Lay (vR p) (vW p) s) (h32 : inB (vW p) (sc (oSB + 32)) 1 = true) + (h33 : inB (vW p) (sc (oSB + 33)) 1 = true) (x y : Nat) (hx : x < 256) (hy : y < 256) : + WP isa (.block (setB (sc (oSB + 32)) x ++ setB (sc (oSB + 33)) y)) s fun s' => PPostB s s' wsB ∧ + s'.gpr .r15 = s.gpr .r15 ∧ + s'.mem = (s.mem.writeW (pa s (sc oSB) + BitVec.ofNat 64 32) (BitVec.ofNat 8 x)).writeW + (pa s (sc oSB) + BitVec.ofNat 64 33) (BitVec.ofNat 8 y) := + setB2At_ok L oSB h32 h33 x y hx hy theorem kl_le : ∀ p ∈ params, p.ℓ ≤ 7 ∧ p.k ≤ 8 := by decide @@ -114,7 +126,7 @@ theorem aOne_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ have hc8 : c < 8 := by omega simp only [aChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, Bool.or_eq_true, Bool.not_eq_true', decide_eq_false_iff_not, Nat.not_lt] at hck - obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, hwa⟩, kB⟩, kA⟩, kSB⟩, kSA⟩, kE⟩ := hck + obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, hwa⟩, kB⟩, kA⟩, kSB⟩, kSA⟩, kE⟩, k4⟩ := hck obtain ⟨tB, hB, zB⟩ := keepChk_spec kB obtain ⟨tA, hA, zA⟩ := keepChk_spec kA have L := hs.t.lay hp hv @@ -136,7 +148,9 @@ theorem aOne_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ rw [hP₂.pa (show Reg.rbx ∈ bases by decide), pA, show 20 + 8 * r + c = 20 + (8 * r + c) by omega] refine ⟨t₁.step hp hv hP₂ tA, L₁.keepHint hP₂ hA (L.keepHint hP₁ hB hs.hint), fun i hi => L₁.keepPoly hP₂ (zA i hi) (L.keepPoly hP₁ (zB i hi) (hs.z i hi)), - by rw [L₁.keepBytes hP₂ kSA, L.keepBytes hP₁ kSB, hs.rho], fun r' hr' c' hc' hd => ?_, + by rw [L₁.keepBytes hP₂ kSA, L.keepBytes hP₁ kSB, hs.rho], + fun k hk => by rw [L₁.keepBytes hP₂ (k4 k hk).2, L.keepBytes hP₁ (k4 k hk).1, hs.rho4 k hk], + fun r' hr' c' hc' hd => ?_, ⟨q && (rr == 1), ?_, fun hq r' hr' c' hc' hd => ?_, fun hq => ?_⟩⟩ · rcases done_succ.mp hd with hd | ⟨rfl, rfl⟩ · have := kE r' hr' c' hc' @@ -180,22 +194,193 @@ theorem S3.next {p : Params} {h : List (Vector Bool n)} {r : Nat} {σ s : State} have e : ∀ r' c', c' < p.ℓ → (Done r p.ℓ r' c' ↔ Done (r + 1) 0 r' c') := fun r' c' hc => by unfold Done; omega obtain ⟨q, h15, hok, hbad⟩ := hs.ok - refine ⟨hs.t, hs.hint, hs.z, hs.rho, fun r' hr' c' hc' hd => hs.red r' hr' c' hc' ((e r' c' hc').mpr hd), + refine ⟨hs.t, hs.hint, hs.z, hs.rho, hs.rho4, fun r' hr' c' hc' hd => hs.red r' hr' c' hc' ((e r' c' hc').mpr hd), q, h15, fun hq r' hr' c' hc' hd => hok hq r' hr' c' hc' ((e r' c' hc').mpr hd), fun hq => ?_⟩ obtain ⟨r', hr', c', hc', hd, hn⟩ := hbad hq exact ⟨r', hr', c', hc', (e r' c' hc').mp hd, hn⟩ +/-! ## Four entries at a time -/ + +/-- A piece writing `ws` keeps the entries done before `(r, c)` but those `ex` says, and `S3`'s other facts. -/ +def s3Chk (p : Params) (r c : Nat) (ex : Nat → Nat → Bool) (ws : List (Ptr × Nat)) : Bool := + keepChk p ws && keepB (vB p) ws (sc oSB) 32 && (List.range 4).all (fun k => keepB (vB p) ws (sc (oSB4 + 34 * k)) 32) && + (List.range p.k).all (fun r' => (List.range p.ℓ).all fun c' => + !(decide (r' < r) || (r' == r && decide (c' < c))) || ex r' c' || keepB (vB p) ws (pA r' c') 1024) + +theorem s3Chk_spec {p : Params} {r c : Nat} {ex : Nat → Nat → Bool} {ws : List (Ptr × Nat)} + (h : s3Chk p r c ex ws = true) : + keepChk p ws = true ∧ keepB (vB p) ws (sc oSB) 32 = true ∧ + (∀ k < 4, keepB (vB p) ws (sc (oSB4 + 34 * k)) 32 = true) ∧ + ∀ r' < p.k, ∀ c' < p.ℓ, Done r c r' c' → ex r' c' = false → keepB (vB p) ws (pA r' c') 1024 = true := by + simp only [s3Chk, Bool.and_eq_true] at h + obtain ⟨⟨⟨h1, h2⟩, h3⟩, h4⟩ := h + refine ⟨h1, h2, fun k hk => List.all_eq_true.mp h3 k (List.mem_range.mpr hk), fun r' hr' c' hc' hd hx => ?_⟩ + have := List.all_eq_true.mp (List.all_eq_true.mp h4 r' (List.mem_range.mpr hr')) c' (List.mem_range.mpr hc') + have hd' : (decide (r' < r) || (r' == r && decide (c' < c))) = true := by + unfold Done at hd + rcases hd with h | ⟨rfl, h⟩ <;> simp [h] + rw [hd', hx] at this + simpa using this + +/-- `S3` across a piece that writes `ws`, which `s3Chk` says keeps it. -/ +theorem S3.keep {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {r c : Nat} + {s s' : State} (hs : S3 p h r c σ s) {ws : List (Ptr × Nat)} (hP : PPostB s s' ws) + (h15 : s'.gpr .r15 = s.gpr .r15) (hc : s3Chk p r c (fun _ _ => false) ws = true) : S3 p h r c σ s' := by + obtain ⟨hk, kSB, k4, kE⟩ := s3Chk_spec hc + obtain ⟨tk, hh, zk⟩ := keepChk_spec hk + have L := hs.t.lay hp hv + obtain ⟨q, hq, hok, hbad⟩ := hs.ok + refine ⟨hs.t.step hp hv hP tk, L.keepHint hP hh hs.hint, fun i hi => L.keepPoly hP (zk i hi) (hs.z i hi), + by rw [L.keepBytes hP kSB, hs.rho], fun k hk => by rw [L.keepBytes hP (k4 k hk), hs.rho4 k hk], + fun r' hr' c' hc' hd => L.keepRed hP (kE r' hr' c' hc' hd rfl) (hs.red r' hr' c' hc' hd), + q, by rw [h15, hq], fun hq' r' hr' c' hc' hd => ?_, hbad⟩ + obtain ⟨b, hb⟩ := hok hq' r' hr' c' hc' hd + exact ⟨b, by rw [hb, L.keepPolyAt hP (kE r' hr' c' hc' hd rfl)]⟩ + +/-- The two bytes of seed `j` of `SB4`. -/ +abbrev wsS (j : Nat) : List (Ptr × Nat) := [(sc (oSB4 + 34 * j + 32), 1), (sc (oSB4 + 34 * j + 33), 1)] + +/-- What setting the bytes of seed `j` needs, after `(r, c)`. -/ +def slotChk (p : Params) (r c j : Nat) : Bool := + inB (vW p) (sc (oSB4 + 34 * j + 32)) 1 && inB (vW p) (sc (oSB4 + 34 * j + 33)) 1 && + s3Chk p r c (fun _ _ => false) (wsS j) && + (List.range j).all (fun k => keepB (vB p) (wsS j) (sc (oSB4 + 34 * k)) 34) + +/-- After the bytes of the first `j` seeds of `SB4` for the entries `(r, c₀ + k)`. -/ +structure GS (p : Params) (h : List (Vector Bool n)) (r c c₀ j : Nat) (σ st : State) : Prop where + s3 : S3 p h r c σ st + done : ∀ k < j, bytesAt st.mem (pa st (sc (oSB4 + 34 * k))) 34 = aSeed (vPk p σ) r (c₀ + k) + +theorem slot_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} + {r c c₀ j : Nat} (hr : r < 256) (hj : j < 4) (hx : c₀ + j < 256) (hck : slotChk p r c j = true) {s : State} + (hs : GS p h r c c₀ j σ s) : WP isa (.block (setSR r c₀ j)) s (GS p h r c c₀ (j + 1) σ) := by + simp only [slotChk, Bool.and_eq_true, List.all_eq_true, List.mem_range] at hck + obtain ⟨⟨⟨h32, h33⟩, h3⟩, hk⟩ := hck + have L := hs.s3.t.lay hp hv + unfold setSR + refine WP.mono (setB2At_ok L (oSB4 + 34 * j) h32 h33 (c₀ + j) r hx hr) fun s₁ ⟨hP₁, h15₁, hm₁⟩ => + ⟨hs.s3.keep hp hv hP₁ h15₁ h3, fun k hk' => ?_⟩ + rcases (by omega : k < j ∨ k = j) with hk' | rfl + · rw [L.keepBytes hP₁ (hk k hk'), hs.done k hk'] + · rw [hP₁.pa (show Reg.rbx ∈ bases by decide), hm₁, seed_bytes, hs.s3.rho4 k hj, aSeed, integerToBytes_one, integerToBytes_one] + + +/-- Whether `(r', c')` is one of the four entries from `(r, c₀)`. -/ +abbrev inGrp (r c₀ r' c' : Nat) : Bool := r' == r && decide (c₀ ≤ c') && decide (c' < c₀ + 4) + +/-- The facts about the parameters the entries `(r, c₀), …, (r, c₀ + 3)` need, after `(r, c)`. -/ +def gChk (p : Params) (r c₀ c : Nat) : Bool := + (List.range 4).all (fun j => slotChk p r c j) && rej4Chk (vB p) (vW p) (pA r c₀) (sc (oR4 p)) && + inB (vB p) (pA r c₀) 4096 && inB (vW p) (pA r c₀) 4096 && + s3Chk p r c (inGrp r c₀) [(pA r c₀, 4096), (sc (oR4 p), 8192)] && + decide (c₀ + 4 ≤ p.ℓ) && decide (c₀ ≤ c) && decide (c ≤ c₀ + 4) + +theorem gChk_all : ∀ p ∈ params, ∀ r < p.k, gChk p r 0 0 = true ∧ (p.ℓ = 7 → gChk p r 3 4 = true) := by decide + +theorem pa_poly4 (s : State) (r c k : Nat) : poly4 (pa s (pA r c)) k = pa s (pA r (c + k)) := by + unfold poly4 + simp only [pa] + rw [BitVec.add_assoc, ← BitVec.ofNat_add, show oP (20 + 8 * r + c) + 1024 * k = oP (20 + 8 * r + (c + k)) by + simp only [oP]; omega] + +theorem aGrp_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) + {h : List (Vector Bool n)} {r c₀ c : Nat} (hr : r < p.k) (hck : gChk p r c₀ c = true) {s : State} + (hs : S3 p h r c σ s) : WP isa (aGrp P p r c₀) s (S3 p h r (c₀ + 4) σ) := by + have hkl := kl_le p hp + simp only [gChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, decide_eq_true_eq] at hck + obtain ⟨⟨⟨⟨⟨⟨⟨hsl, hrej⟩, hin⟩, hwa⟩, hG⟩, hl⟩, hc₀⟩, hc4⟩ := hck + obtain ⟨hk, kSB, k4, kE⟩ := s3Chk_spec hG + obtain ⟨tk, hh, zk⟩ := keepChk_spec hk + unfold aGrp + refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 0 (by decide)) + ⟨hs, fun _ h => absurd h (by omega)⟩) fun _ g₁ => ?_) + refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 1 (by decide)) g₁) fun _ g₂ => ?_) + refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 2 (by decide)) g₂) fun _ g₃ => ?_) + refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 3 (by decide)) g₃) fun s₄ g₄ => ?_) + have L₄ := g₄.s3.t.lay hp hv + obtain ⟨q, h15, hok, hbad⟩ := g₄.s3.ok + have hseed : ∀ k < 4, seed4 s₄.mem (pa s₄ (sc oSB4)) k = aSeed (vPk p σ) r (c₀ + k) := fun k hk => by + unfold seed4 + rw [show pa s₄ (sc oSB4) + BitVec.ofNat 64 (34 * k) = pa s₄ (sc (oSB4 + 34 * k)) by + simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add]] + exact g₄.done k hk + refine WP.mono (sampled4_ok L₄ hin hwa h15 (List.mem_cons_self ..) + (F := fun k b => rejNTTPoly b.rejNTT (aSeed (vPk p σ) r (c₀ + k))) + (WP.mono (rej4At_ok C.rej4 L₄ hrej) fun s' ⟨hP, h15', hred, ho⟩ => ⟨hP, h15', hred, ?_⟩)) + fun s₅ ⟨hP₅, hred₅, rr, hrr, h15₅, hs1, hs0⟩ => ?_ + · rcases ho with ⟨h1, hb⟩ | ⟨h0, k, hk, hn⟩ + · exact .inl ⟨h1, fun k hk => by rw [← hseed k hk]; exact hb k hk⟩ + · exact .inr ⟨h0, k, hk, by rw [← hseed k hk]; exact hn⟩ + have e₅ : ∀ k, poly4 (pa s₄ (pA r c₀)) k = pa s₅ (pA r (c₀ + k)) := fun k => by + rw [pa_poly4, hP₅.pa (show Reg.rbx ∈ bases by decide)] + have hgrp : ∀ c', inGrp r c₀ r c' = true → c₀ ≤ c' ∧ c' < c₀ + 4 := fun c' hg => by + simp only [inGrp, Bool.and_eq_true, beq_self_eq_true, decide_eq_true_eq, true_and] at hg; exact hg + have hout : ∀ r' c', Done r (c₀ + 4) r' c' → inGrp r c₀ r' c' = false → Done r c r' c' := fun r' c' hd hx => by + unfold Done at hd ⊢ + rcases hd with hd | ⟨rfl, hd⟩ + · exact .inl hd + · refine .inr ⟨rfl, ?_⟩ + simp only [inGrp, beq_self_eq_true, Bool.true_and, Bool.and_eq_false_iff, decide_eq_false_iff_not] at hx + omega + refine ⟨g₄.s3.t.step hp hv hP₅ tk, L₄.keepHint hP₅ hh g₄.s3.hint, + fun i hi => L₄.keepPoly hP₅ (zk i hi) (g₄.s3.z i hi), by rw [L₄.keepBytes hP₅ kSB, g₄.s3.rho], + fun k hk => by rw [L₄.keepBytes hP₅ (k4 k hk), g₄.s3.rho4 k hk], fun r' hr' c' hc' hd => ?_, + q && (rr == 1), ?_, fun hq r' hr' c' hc' hd => ?_, fun hq => ?_⟩ + · cases hx : inGrp r c₀ r' c' + · exact L₄.keepRed hP₅ (kE r' hr' c' hc' (hout r' c' hd hx) hx) (g₄.s3.red r' hr' c' hc' (hout r' c' hd hx)) + · have e : r' = r := by simp only [inGrp, Bool.and_eq_true, beq_iff_eq] at hx; exact hx.1.1 + subst e + obtain ⟨g1, g2⟩ := hgrp c' hx + have := hred₅ (c' - c₀) (by omega) + rwa [e₅, show c₀ + (c' - c₀) = c' by omega] at this + · rw [h15₅] + exact flag_congr (by cases q <;> simp) + · simp only [Bool.and_eq_true, beq_iff_eq] at hq + cases hx : inGrp r c₀ r' c' + · obtain ⟨b, hb⟩ := hok hq.1 r' hr' c' hc' (hout r' c' hd hx) + exact ⟨b, by rw [hb, L₄.keepPolyAt hP₅ (kE r' hr' c' hc' (hout r' c' hd hx) hx)]⟩ + · have e : r' = r := by simp only [inGrp, Bool.and_eq_true, beq_iff_eq] at hx; exact hx.1.1 + subst e + obtain ⟨g1, g2⟩ := hgrp c' hx + obtain ⟨b, hb⟩ := hs1 hq.2 (c' - c₀) (by omega) + refine ⟨b, ?_⟩ + rwa [e₅, show c₀ + (c' - c₀) = c' by omega] at hb + · cases hq' : q + · obtain ⟨r', hr', c', hc', hd, hn⟩ := hbad hq' + refine ⟨r', hr', c', hc', ?_, hn⟩ + unfold Done at hd ⊢; omega + · rw [hq'] at hq + have h0 : rr = 0 := by + rcases hrr with h1 | h0 + · rw [h1] at hq; cases hq + · exact h0 + obtain ⟨k, hk, hn⟩ := hs0 h0 + exact ⟨r, hr, c₀ + k, by omega, by unfold Done; omega, hn⟩ + theorem aRow_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {r : Nat} (hr : r < p.k) {s : State} (hs : S3 p h r 0 σ s) : WP isa (aRow P p r) s (S3 p h (r + 1) 0 σ) := by + have hkl := kl_le p hp + have hg := gChk_all p hp r hr + have hl : p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by + have : ∀ p ∈ params, p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by decide + exact this p hp unfold aRow - refine WP.mono (seqR_ok (I := fun e => S3 p h r (e - 8 * r) σ) p.ℓ (8 * r) (fun e he he' st hst => ?_) s - (by rw [Nat.sub_self]; exact hs)) fun st hst => ?_ - · have := aOne_ok C hp hv hr (c := e - 8 * r) (by omega) hst - rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this - exact this - · rw [show 8 * r + p.ℓ - 8 * r = p.ℓ by omega] at hst - exact hst.next + refine WP.seq (WP.mono (aGrp_ok C hp hv hr hg.1 hs) fun s₁ h₁ => ?_) + rw [Nat.zero_add] at h₁ + by_cases h7 : p.ℓ = 7 + · rw [ite_eq_left h7] + refine WP.mono (aGrp_ok C hp hv hr (hg.2 h7) h₁) fun s₂ h₂ => ?_ + rw [show 3 + 4 = p.ℓ by omega] at h₂ + exact h₂.next + · rw [ite_eq_right h7] + refine WP.mono (seqR_ok (I := fun e => S3 p h r (e - 8 * r) σ) (p.ℓ - 4) (8 * r + 4) + (fun e he he' st hst => ?_) s₁ (by rw [show 8 * r + 4 - 8 * r = 4 by omega]; exact h₁)) fun st hst => ?_ + · have := aOne_ok C hp hv hr (c := e - 8 * r) (by omega) hst + rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this + exact this + · rw [show 8 * r + 4 + (p.ℓ - 4) - 8 * r = p.ℓ by omega] at hst + exact hst.next /-- After the samplers, with the hint `h`. -/ structure S4 (p : Params) (h : List (Vector Bool n)) (σ st : State) : Prop where @@ -223,9 +408,18 @@ def sChk (p : Params) : Bool := theorem sChk_all : ∀ p ∈ params, sChk p = true := by decide +/-- After `ρ` is copied to `SB` and the first `j` seeds of `SB4`. -/ +structure RhoS (p : Params) (h : List (Vector Bool n)) (j : Nat) (σ st : State) : Prop where + t : T p σ st + hint : HintIs st.mem (pa st (pH 0)) p.k h + z : ∀ i < p.ℓ, PolyIs st.mem (pa st (pZ i)) (toRq (vZ p (vSig p σ) i)) + rho : bytesAt st.mem (pa st (sc oSB)) 32 = vRho (vPk p σ) + rho4 : ∀ k < j, bytesAt st.mem (pa st (sc (oSB4 + 34 * k))) 32 = vRho (vPk p σ) + f15 : st.gpr .r15 = flag True + theorem copyRho_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {s : State} (hs : S2 p h p.ℓ σ s) (h15 : s.gpr .r15 = flag True) : - WP isa (copy (sc oSB) (.rbp, 0) 32) s (S3 p h 0 0 σ) := by + WP isa (copy (sc oSB) (.rbp, 0) 32) s (RhoS p h 0 σ) := by have hc := sChk_all p hp simp only [sChk, Bool.and_eq_true, decide_eq_true_eq, List.all_eq_true, List.mem_range] at hc obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨c1, c2⟩, c3⟩, c4⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hc @@ -234,11 +428,53 @@ theorem copyRho_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ refine WP.mono (copy_ok L (by decide) c1 c2) fun s₁ ⟨hP₁, h15₁, hcp⟩ => ?_ have t₁ := hs.t.step hp hv hP₁ t3 refine ⟨t₁, L.keepHint hP₁ h3 hs.hint, fun i hi => L.keepPoly hP₁ (z3 i hi) (hs.z i hi), ?_, - fun r' _ c' _ hd => absurd hd (by unfold Done; omega), true, by rw [h15₁, h15]; exact flag_congr (by simp), - fun _ r' _ c' _ hd => absurd hd (by unfold Done; omega), fun hq => absurd hq (by simp)⟩ + fun _ h => absurd h (Nat.not_lt_zero _), by rw [h15₁, h15]⟩ rw [hP₁.pa (show Reg.rbx ∈ bases by decide), hcp, hs.t.pkSlice c4, List.drop_zero] rfl +/-- What copying `ρ` to seed `j` of `SB4` needs. -/ +def rChk (p : Params) (j : Nat) : Bool := + sepB (vB p) (.rbp, 0) 32 (sc (oSB4 + 34 * j)) 32 && inB (vW p) (sc (oSB4 + 34 * j)) 32 && + keepChk p [(sc (oSB4 + 34 * j), 32)] && keepB (vB p) [(sc (oSB4 + 34 * j), 32)] (sc oSB) 32 && + (List.range j).all (fun k => keepB (vB p) [(sc (oSB4 + 34 * j), 32)] (sc (oSB4 + 34 * k)) 32) + +theorem rChk_all : ∀ p ∈ params, ∀ j < 4, rChk p j = true := by decide + +theorem copyK_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {j : Nat} + (hj : j < 4) {s : State} (hs : RhoS p h j σ s) : + WP isa (copy (sc (oSB4 + 34 * j)) (.rbp, 0) 32) s (RhoS p h (j + 1) σ) := by + have hc := rChk_all p hp j hj + simp only [rChk, Bool.and_eq_true, List.all_eq_true, List.mem_range] at hc + obtain ⟨⟨⟨⟨c1, c2⟩, c3⟩, c4⟩, c5⟩ := hc + have hpk : 32 ≤ p.pkLen := by + have := sChk_all p hp + simp only [sChk, Bool.and_eq_true, decide_eq_true_eq] at this + exact this.1.1.1.1.1.1.1.2 + obtain ⟨t3, h3, z3⟩ := keepChk_spec c3 + have L := hs.t.lay hp hv + refine WP.mono (copy_ok L (by decide) c1 c2) fun s₁ ⟨hP₁, h15₁, hcp⟩ => ?_ + refine ⟨hs.t.step hp hv hP₁ t3, L.keepHint hP₁ h3 hs.hint, fun i hi => L.keepPoly hP₁ (z3 i hi) (hs.z i hi), + by rw [L.keepBytes hP₁ c4, hs.rho], fun k hk => ?_, by rw [h15₁, hs.f15]⟩ + rcases (by omega : k < j ∨ k = j) with hk | rfl + · rw [L.keepBytes hP₁ (c5 k hk), hs.rho4 k hk] + · rw [hP₁.pa (show Reg.rbx ∈ bases by decide), hcp, hs.t.pkSlice hpk, List.drop_zero] + rfl + +theorem RhoS.s3 {p : Params} {h : List (Vector Bool n)} {σ s : State} (r : RhoS p h 4 σ s) : S3 p h 0 0 σ s := + ⟨r.t, r.hint, r.z, r.rho, r.rho4, fun r' _ c' _ hd => absurd hd (by unfold Done; omega), + true, by rw [r.f15]; exact flag_congr (by simp), fun _ r' _ c' _ hd => absurd hd (by unfold Done; omega), + fun hq => absurd hq (by simp)⟩ + +theorem rhos_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) + {h : List (Vector Bool n)} {s : State} (hs : S2 p h p.ℓ σ s) (h15 : s.gpr .r15 = flag True) : + WP isa rhos s (S3 p h 0 0 σ) := by + unfold rhos + refine WP.seq (WP.mono (copyRho_ok hp hv hs h15) fun s₀ r₀ => ?_) + refine WP.seq (WP.mono (copyK_ok hp hv (j := 0) (by decide) r₀) fun s₁ r₁ => ?_) + refine WP.seq (WP.mono (copyK_ok hp hv (j := 1) (by decide) r₁) fun s₂ r₂ => ?_) + refine WP.seq (WP.mono (copyK_ok hp hv (j := 2) (by decide) r₂) fun s₃ r₃ => ?_) + exact WP.mono (copyK_ok hp hv (j := 3) (by decide) r₃) fun _ r₄ => r₄.s3 + theorem ballStage_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {s₂ : State} (hs₂ : S3 p h p.k 0 σ s₂) : WP isa (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC) s₂ (S4 p h σ) := by @@ -279,7 +515,7 @@ theorem samples_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {h : List (Vector Bool n)} {s : State} (hs : S2 p h p.ℓ σ s) (h15 : s.gpr .r15 = flag True) : WP isa (samples P p) s (S4 p h σ) := by unfold samples - refine WP.seq (WP.mono (copyRho_ok hp hv hs h15) fun s₁ s₁3 => ?_) + refine WP.seq (WP.mono (rhos_ok hp hv hs h15) fun s₁ s₁3 => ?_) refine WP.seq (WP.mono (seqR_ok (I := fun r => S3 p h r 0 σ) p.k 0 (fun r _ hr st hst => aRow_ok C hp hv (by omega) hst) s₁ s₁3) fun s₂ hs₂ => ?_) rw [Nat.zero_add] at hs₂ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Top.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Top.lean index e623e7872..6369ce2eb 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Top.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Top.lean @@ -6,7 +6,7 @@ import VerifiedGarbage.Spec.MlDsa.Contract # ML-DSA verification on x86-64: the contract, the layout and the invariant Untrusted: everything here is checked by Lean. The precondition of the -shared contract `verifyContract p X86_64.abi 24`, spelled out (`VPre`); the +shared contract `verifyContract p X86_64.abi 32`, spelled out (`VPre`); the layout of the function's buffers (`pk`, `mu`, `sig` read in `rbp`, `r12`, `r13`; `scratch` written, in `rbx`: `vR p`, `vW p`); what holds throughout (`T`: the permissions and stack pointer of entry, the pointers, the caller's @@ -28,9 +28,9 @@ def params : List Params := [mlDsa44, mlDsa65, mlDsa87] /-- The size of `scratch` in bytes. -/ abbrev scrLen (p : Params) : Nat := scratchWords p * 8 -/-- The precondition of `verifyContract p X86_64.abi 24`. -/ +/-- The precondition of `verifyContract p X86_64.abi 32`. -/ structure VPre (p : Params) (σ : State) : Prop where - sp : 24 ≤ (σ.gpr .rsp).toNat + sp : 32 ≤ (σ.gpr .rsp).toNat rd : σ.rd = [⟨σ.gpr .rdi, p.pkLen⟩, ⟨σ.gpr .rsi, 64⟩, ⟨σ.gpr .rdx, p.sigLen⟩] wr : σ.wr = [⟨σ.gpr .rcx, scrLen p⟩] d1 : Region.Disjoint ⟨σ.gpr .rdi, p.pkLen⟩ ⟨σ.gpr .rcx, scrLen p⟩ @@ -40,10 +40,10 @@ structure VPre (p : Params) (σ : State) : Prop where r2 : Region.Disjoint ⟨σ.gpr .rsp, 8⟩ ⟨σ.gpr .rsi, 64⟩ r3 : Region.Disjoint ⟨σ.gpr .rsp, 8⟩ ⟨σ.gpr .rdx, p.sigLen⟩ r4 : Region.Disjoint ⟨σ.gpr .rsp, 8⟩ ⟨σ.gpr .rcx, scrLen p⟩ - k1 : (below (σ.gpr .rsp) 24).Disjoint ⟨σ.gpr .rdi, p.pkLen⟩ - k2 : (below (σ.gpr .rsp) 24).Disjoint ⟨σ.gpr .rsi, 64⟩ - k3 : (below (σ.gpr .rsp) 24).Disjoint ⟨σ.gpr .rdx, p.sigLen⟩ - k4 : (below (σ.gpr .rsp) 24).Disjoint ⟨σ.gpr .rcx, scrLen p⟩ + k1 : (below (σ.gpr .rsp) 32).Disjoint ⟨σ.gpr .rdi, p.pkLen⟩ + k2 : (below (σ.gpr .rsp) 32).Disjoint ⟨σ.gpr .rsi, 64⟩ + k3 : (below (σ.gpr .rsp) 32).Disjoint ⟨σ.gpr .rdx, p.sigLen⟩ + k4 : (below (σ.gpr .rsp) 32).Disjoint ⟨σ.gpr .rcx, scrLen p⟩ n1 : (σ.gpr .rdi).toNat + p.pkLen ≤ 2 ^ 64 n2 : (σ.gpr .rsi).toNat + 64 ≤ 2 ^ 64 n3 : (σ.gpr .rdx).toNat + p.sigLen ≤ 2 ^ 64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Verified.lean index 37d95f25b..f87b8ef96 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Verified.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Verified.lean @@ -70,7 +70,7 @@ def verifySat (p : Params) : State where rd := [⟨0x10000, p.pkLen⟩, ⟨0x20000, 64⟩, ⟨0x30000, p.sigLen⟩] wr := [⟨0x40000, scrLen p⟩] -theorem verify_sat : ∀ p ∈ params, ∃ s, (verifyContract p X86_64.abi 24).pre s := by +theorem verify_sat : ∀ p ∈ params, ∃ s, (verifyContract p X86_64.abi 32).pre s := by intro p hp simp only [params, List.mem_cons, List.not_mem_nil, or_false] at hp rcases hp with rfl | rfl | rfl @@ -78,7 +78,7 @@ theorem verify_sat : ∀ p ∈ params, ∃ s, (verifyContract p X86_64.abi 24).p · sig_implies_sat [verifyContract, verifySig, X86_64.abi, VG.X86_64.argRegs] [verifySat] using verifySat mlDsa65 · sig_implies_sat [verifyContract, verifySig, X86_64.abi, VG.X86_64.argRegs] [verifySat] using verifySat mlDsa87 -theorem verify_implies {p : Params} (hp : p ∈ params) : (verifyK p).Implies (verifyContract p X86_64.abi 24) where +theorem verify_implies {p : Params} (hp : p ∈ params) : (verifyK p).Implies (verifyContract p X86_64.abi 32) where pre s h := by sig_pre [verifyContract, verifySig, X86_64.abi, VG.X86_64.argRegs] at h obtain ⟨a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18⟩ := h @@ -95,9 +95,9 @@ theorem verify_implies {p : Params} (hp : p ∈ params) : (verifyK p).Implies (v exact ⟨hdi, hsi, hdx, hcx, hsp, e₁, e₂, e₃⟩ sat := verify_sat p hp -/-- `verify P p` meets `verifyContract p` with 24 bytes of stack. -/ +/-- `verify P p` meets `verifyContract p` with 32 bytes of stack. -/ theorem verify_verified {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) : - Verified X86_64.target (verify P p) (verifyContract p X86_64.abi 24) := + Verified X86_64.target (verify P p) (verifyContract p X86_64.abi 32) := Verified.of_correct (verify_correct C hp) (verify_ct C hp) (verify_implies hp) end VG.Proof.MlDsa.X86_64.Verify diff --git a/src/asm/x86_64/mldsa.rs b/src/asm/x86_64/mldsa.rs index 541b9d25a..2997f10a4 100644 --- a/src/asm/x86_64/mldsa.rs +++ b/src/asm/x86_64/mldsa.rs @@ -4933,6 +4933,4847 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_rej_ntt_poly(seed: *const [u8; 34] ) } +/// `RejNTTPoly` (FIPS 204 Algorithm 30) four times: for each `k` < 4, writes the element of `T_q` sampled from the SHAKE128 output of the 34 bytes of `*seeds` from byte `34 k` to the 256 coefficients of `*a` from coefficient `256 k` (each less than `q` = 8380417), and returns 1. Returns 0 if the loop reaches its bound, which is at least 894 bytes of SHAKE128 output for each (FIPS 204 Appendix C; this happens with probability about 2^-256 or less): `*a` is then unspecified, and the caller must destroy it and treat the operation as failed. The four are independent, so an implementation may compute them together (e.g. four SHAKE128 instances at once in vector registers). +/// +/// Contract: `VG.Spec.MlDsa.rejNTT4Contract`. Not constant time in the seeds: timing may depend on the pointers and on `*seeds` (public in ML-DSA: the seed `ρ` of the matrix and indices), but not on anything else. +/// +/// It calls `vg_mldsa_rej_ntt_poly` on each seed, and saves its caller's callee-saved registers in `scratch`. +/// +/// # Safety +/// +/// * `seeds` must be valid for reads of 136 bytes. +/// * `a` must be valid for reads and writes of 4096 bytes. +/// * `scratch` must be valid for reads and writes of 8192 bytes. +/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `a` and `scratch` must not overlap each other or `seeds` (distinct Rust objects never do). +/// * None of `seeds`, `a` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_rej_ntt_poly4(seeds: *const [u8; 136], a: *mut [u32; 1024], scratch: *mut [u64; 1024]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rdx+4384], rbx", + "mov QWORD PTR [rdx+4392], rbp", + "mov QWORD PTR [rdx+4400], r12", + "mov QWORD PTR [rdx+4408], r13", + "mov QWORD PTR [rdx+4416], r14", + "mov rbx, rdx", + "mov r12, rdi", + "mov r13, rsi", + "mov r14d, 1", + "mov rdi, r12", + "add rdi, 0", + "mov rsi, r13", + "add rsi, 0", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_rej_ntt_poly}", + "and r14d, eax", + "mov rdi, r12", + "add rdi, 34", + "mov rsi, r13", + "add rsi, 1024", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_rej_ntt_poly}", + "and r14d, eax", + "mov rdi, r12", + "add rdi, 68", + "mov rsi, r13", + "add rsi, 2048", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_rej_ntt_poly}", + "and r14d, eax", + "mov rdi, r12", + "add rdi, 102", + "mov rsi, r13", + "add rsi, 3072", + "mov rdx, rbx", + "add rdx, 6144", + "call {vg_mldsa_rej_ntt_poly}", + "and r14d, eax", + "mov eax, r14d", + "mov r14, QWORD PTR [rbx+4416]", + "mov r13, QWORD PTR [rbx+4408]", + "mov r12, QWORD PTR [rbx+4400]", + "mov rbp, QWORD PTR [rbx+4392]", + "mov rbx, QWORD PTR [rbx+4384]", + "ret", + vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + ) +} + +/// The CPU features `vg_mldsa_rej_ntt_poly4_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_REJ_NTT_POLY4_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// `RejNTTPoly` (FIPS 204 Algorithm 30) four times: for each `k` < 4, writes the element of `T_q` sampled from the SHAKE128 output of the 34 bytes of `*seeds` from byte `34 k` to the 256 coefficients of `*a` from coefficient `256 k` (each less than `q` = 8380417), and returns 1. Returns 0 if the loop reaches its bound, which is at least 894 bytes of SHAKE128 output for each (FIPS 204 Appendix C; this happens with probability about 2^-256 or less): `*a` is then unspecified, and the caller must destroy it and treat the operation as failed. The four are independent, so an implementation may compute them together (e.g. four SHAKE128 instances at once in vector registers). +/// +/// Contract: `VG.Spec.MlDsa.rejNTT4Contract`. Not constant time in the seeds: timing may depend on the pointers and on `*seeds` (public in ML-DSA: the seed `ρ` of the matrix and indices), but not on anything else. +/// +/// It runs the four instances of SHAKE128 at once, in the four 64-bit elements of AVX2 registers (as `vg_mlkem_sample_ntt4_avx2` does), squeezing three blocks of each twice, and runs the loop of `RejNTTPoly` over the 1008 bytes of each seed's output, as `vg_mldsa_rej_ntt_poly` does. It saves its caller's callee-saved registers in `scratch`. +/// +/// # Safety +/// +/// * `seeds` must be valid for reads of 136 bytes. +/// * `a` must be valid for reads and writes of 4096 bytes. +/// * `scratch` must be valid for reads and writes of 8192 bytes. +/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `a` and `scratch` must not overlap each other or `seeds` (distinct Rust objects never do). +/// * None of `seeds`, `a` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_rej_ntt_poly4_avx2(seeds: *const [u8; 136], a: *mut [u32; 1024], scratch: *mut [u64; 1024]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rdx+4384], rbx", + "mov QWORD PTR [rdx+4392], rbp", + "mov QWORD PTR [rdx+4400], r12", + "mov QWORD PTR [rdx+4408], r13", + "mov QWORD PTR [rdx+4416], r14", + "mov rbx, rdx", + "mov r12, rdi", + "mov r13, rsi", + "mov r14d, 1", + "movabs rax, 1", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1600], ymm0", + "movabs rax, 32898", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1632], ymm0", + "movabs rax, -9223372036854742902", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1664], ymm0", + "movabs rax, -9223372034707259392", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1696], ymm0", + "movabs rax, 32907", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1728], ymm0", + "movabs rax, 2147483649", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1760], ymm0", + "movabs rax, -9223372034707259263", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1792], ymm0", + "movabs rax, -9223372036854743031", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1824], ymm0", + "movabs rax, 138", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1856], ymm0", + "movabs rax, 136", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1888], ymm0", + "movabs rax, 2147516425", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1920], ymm0", + "movabs rax, 2147483658", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1952], ymm0", + "movabs rax, 2147516555", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1984], ymm0", + "movabs rax, -9223372036854775669", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2016], ymm0", + "movabs rax, -9223372036854742903", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2048], ymm0", + "movabs rax, -9223372036854743037", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2080], ymm0", + "movabs rax, -9223372036854743038", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2112], ymm0", + "movabs rax, -9223372036854775680", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2144], ymm0", + "movabs rax, 32778", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2176], ymm0", + "movabs rax, -9223372034707292150", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2208], ymm0", + "movabs rax, -9223372034707259263", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2240], ymm0", + "movabs rax, -9223372036854742912", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2272], ymm0", + "movabs rax, 2147483649", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2304], ymm0", + "movabs rax, -9223372034707259384", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2336], ymm0", + "vpxor ymm0, ymm0, ymm0", + "vmovdqu YMMWORD PTR [rbx], ymm0", + "vmovdqu YMMWORD PTR [rbx+32], ymm0", + "vmovdqu YMMWORD PTR [rbx+64], ymm0", + "vmovdqu YMMWORD PTR [rbx+96], ymm0", + "vmovdqu YMMWORD PTR [rbx+128], ymm0", + "vmovdqu YMMWORD PTR [rbx+160], ymm0", + "vmovdqu YMMWORD PTR [rbx+192], ymm0", + "vmovdqu YMMWORD PTR [rbx+224], ymm0", + "vmovdqu YMMWORD PTR [rbx+256], ymm0", + "vmovdqu YMMWORD PTR [rbx+288], ymm0", + "vmovdqu YMMWORD PTR [rbx+320], ymm0", + "vmovdqu YMMWORD PTR [rbx+352], ymm0", + "vmovdqu YMMWORD PTR [rbx+384], ymm0", + "vmovdqu YMMWORD PTR [rbx+416], ymm0", + "vmovdqu YMMWORD PTR [rbx+448], ymm0", + "vmovdqu YMMWORD PTR [rbx+480], ymm0", + "vmovdqu YMMWORD PTR [rbx+512], ymm0", + "vmovdqu YMMWORD PTR [rbx+544], ymm0", + "vmovdqu YMMWORD PTR [rbx+576], ymm0", + "vmovdqu YMMWORD PTR [rbx+608], ymm0", + "vmovdqu YMMWORD PTR [rbx+640], ymm0", + "vmovdqu YMMWORD PTR [rbx+672], ymm0", + "vmovdqu YMMWORD PTR [rbx+704], ymm0", + "vmovdqu YMMWORD PTR [rbx+736], ymm0", + "vmovdqu YMMWORD PTR [rbx+768], ymm0", + "mov rax, QWORD PTR [r12]", + "mov QWORD PTR [rbx], rax", + "mov rax, QWORD PTR [r12+8]", + "mov QWORD PTR [rbx+32], rax", + "mov rax, QWORD PTR [r12+16]", + "mov QWORD PTR [rbx+64], rax", + "mov rax, QWORD PTR [r12+24]", + "mov QWORD PTR [rbx+96], rax", + "movzx eax, BYTE PTR [r12+32]", + "mov BYTE PTR [rbx+128], al", + "movzx eax, BYTE PTR [r12+33]", + "mov BYTE PTR [rbx+129], al", + "mov rax, QWORD PTR [r12+34]", + "mov QWORD PTR [rbx+8], rax", + "mov rax, QWORD PTR [r12+42]", + "mov QWORD PTR [rbx+40], rax", + "mov rax, QWORD PTR [r12+50]", + "mov QWORD PTR [rbx+72], rax", + "mov rax, QWORD PTR [r12+58]", + "mov QWORD PTR [rbx+104], rax", + "movzx eax, BYTE PTR [r12+66]", + "mov BYTE PTR [rbx+136], al", + "movzx eax, BYTE PTR [r12+67]", + "mov BYTE PTR [rbx+137], al", + "mov rax, QWORD PTR [r12+68]", + "mov QWORD PTR [rbx+16], rax", + "mov rax, QWORD PTR [r12+76]", + "mov QWORD PTR [rbx+48], rax", + "mov rax, QWORD PTR [r12+84]", + "mov QWORD PTR [rbx+80], rax", + "mov rax, QWORD PTR [r12+92]", + "mov QWORD PTR [rbx+112], rax", + "movzx eax, BYTE PTR [r12+100]", + "mov BYTE PTR [rbx+144], al", + "movzx eax, BYTE PTR [r12+101]", + "mov BYTE PTR [rbx+145], al", + "mov rax, QWORD PTR [r12+102]", + "mov QWORD PTR [rbx+24], rax", + "mov rax, QWORD PTR [r12+110]", + "mov QWORD PTR [rbx+56], rax", + "mov rax, QWORD PTR [r12+118]", + "mov QWORD PTR [rbx+88], rax", + "mov rax, QWORD PTR [r12+126]", + "mov QWORD PTR [rbx+120], rax", + "movzx eax, BYTE PTR [r12+134]", + "mov BYTE PTR [rbx+152], al", + "movzx eax, BYTE PTR [r12+135]", + "mov BYTE PTR [rbx+153], al", + "mov eax, 31", + "mov BYTE PTR [rbx+130], al", + "mov BYTE PTR [rbx+138], al", + "mov BYTE PTR [rbx+146], al", + "mov BYTE PTR [rbx+154], al", + "mov eax, 128", + "mov BYTE PTR [rbx+647], al", + "mov BYTE PTR [rbx+655], al", + "mov BYTE PTR [rbx+663], al", + "mov BYTE PTR [rbx+671], al", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "20:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 20b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2368], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2376], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2384], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2392], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2400], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2408], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2416], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2424], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2432], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2440], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2448], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2456], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2464], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2472], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2480], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2488], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2496], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2504], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2512], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2520], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2528], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+2872], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+2880], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+2888], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+2896], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+2904], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+2912], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+2920], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+2928], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+2936], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+2944], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+2952], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+2960], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+2968], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+2976], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+2984], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+2992], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3000], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3008], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3016], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3024], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3032], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3376], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3384], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3392], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3400], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3408], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3416], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3424], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3432], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3440], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3448], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3456], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3464], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3472], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3480], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3488], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3496], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3504], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3512], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3520], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3528], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3536], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+3880], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+3888], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+3896], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+3904], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+3912], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+3920], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+3928], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+3936], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+3944], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+3952], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+3960], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+3968], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+3976], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+3984], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+3992], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4000], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4008], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4016], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4024], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4032], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4040], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "21:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 21b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2536], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2544], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2552], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2560], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2568], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2576], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2584], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2592], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2600], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2608], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2616], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2624], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2632], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2640], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2648], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2656], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2664], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2672], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2680], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2688], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2696], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3040], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3048], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3056], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3064], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3072], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3080], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3088], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3096], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3104], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3112], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3120], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3128], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3136], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3144], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3152], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3160], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3168], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3176], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3184], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3192], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3200], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3544], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3552], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3560], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3568], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3576], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3584], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3592], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3600], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3608], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3616], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3624], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3632], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3640], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3648], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3656], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3664], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3672], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3680], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3688], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3696], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3704], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4048], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4056], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4064], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4072], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4080], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4088], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4096], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4104], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4112], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4120], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4128], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4136], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4144], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4152], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4160], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4168], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4176], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4184], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4192], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4200], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4208], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "22:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 22b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2704], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2712], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2720], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2728], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2736], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2744], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2752], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2760], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2768], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2776], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2784], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2792], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2800], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2808], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2816], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2824], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2832], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2840], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2848], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2856], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2864], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3208], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3216], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3224], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3232], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3240], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3248], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3256], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3264], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3272], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3280], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3288], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3296], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3304], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3312], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3320], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3328], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3336], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3344], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3352], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3360], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3368], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3712], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3720], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3728], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3736], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3744], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3752], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3760], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3768], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3776], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3784], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3792], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3800], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3808], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3816], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3824], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3832], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3840], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3848], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3856], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3864], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3872], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4216], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4224], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4232], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4240], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4248], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4256], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4264], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4272], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4280], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4288], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4296], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4304], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4312], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4320], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4328], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4336], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4344], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4352], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4360], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4368], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4376], rax", + "mov eax, 0", + "mov QWORD PTR [rbx+4424], rax", + "mov QWORD PTR [rbx+4432], rax", + "mov QWORD PTR [rbx+4440], rax", + "mov QWORD PTR [rbx+4448], rax", + "mov rsi, rbx", + "add rsi, 2368", + "mov rbp, r13", + "add rbp, 0", + "mov rdi, QWORD PTR [rbx+4424]", + "mov ecx, 168", + "23:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 24f", + "jmp 25f", + "24:", + "cmp r8d, 8380417", + "jb 26f", + "jmp 27f", + "26:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "27:", + "25:", + "add rsi, 3", + "sub rcx, 1", + "jne 23b", + "mov QWORD PTR [rbx+4424], rdi", + "mov rsi, rbx", + "add rsi, 2872", + "mov rbp, r13", + "add rbp, 1024", + "mov rdi, QWORD PTR [rbx+4432]", + "mov ecx, 168", + "28:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 29f", + "jmp 210f", + "29:", + "cmp r8d, 8380417", + "jb 211f", + "jmp 212f", + "211:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "212:", + "210:", + "add rsi, 3", + "sub rcx, 1", + "jne 28b", + "mov QWORD PTR [rbx+4432], rdi", + "mov rsi, rbx", + "add rsi, 3376", + "mov rbp, r13", + "add rbp, 2048", + "mov rdi, QWORD PTR [rbx+4440]", + "mov ecx, 168", + "213:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 214f", + "jmp 215f", + "214:", + "cmp r8d, 8380417", + "jb 216f", + "jmp 217f", + "216:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "217:", + "215:", + "add rsi, 3", + "sub rcx, 1", + "jne 213b", + "mov QWORD PTR [rbx+4440], rdi", + "mov rsi, rbx", + "add rsi, 3880", + "mov rbp, r13", + "add rbp, 3072", + "mov rdi, QWORD PTR [rbx+4448]", + "mov ecx, 168", + "218:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 219f", + "jmp 220f", + "219:", + "cmp r8d, 8380417", + "jb 221f", + "jmp 222f", + "221:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "222:", + "220:", + "add rsi, 3", + "sub rcx, 1", + "jne 218b", + "mov QWORD PTR [rbx+4448], rdi", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "223:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 223b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2368], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2376], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2384], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2392], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2400], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2408], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2416], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2424], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2432], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2440], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2448], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2456], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2464], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2472], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2480], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2488], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2496], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2504], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2512], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2520], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2528], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+2872], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+2880], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+2888], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+2896], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+2904], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+2912], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+2920], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+2928], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+2936], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+2944], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+2952], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+2960], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+2968], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+2976], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+2984], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+2992], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3000], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3008], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3016], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3024], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3032], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3376], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3384], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3392], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3400], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3408], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3416], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3424], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3432], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3440], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3448], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3456], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3464], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3472], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3480], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3488], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3496], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3504], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3512], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3520], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3528], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3536], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+3880], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+3888], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+3896], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+3904], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+3912], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+3920], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+3928], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+3936], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+3944], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+3952], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+3960], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+3968], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+3976], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+3984], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+3992], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4000], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4008], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4016], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4024], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4032], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4040], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "224:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 224b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2536], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2544], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2552], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2560], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2568], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2576], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2584], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2592], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2600], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2608], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2616], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2624], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2632], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2640], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2648], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2656], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2664], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2672], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2680], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2688], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2696], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3040], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3048], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3056], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3064], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3072], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3080], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3088], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3096], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3104], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3112], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3120], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3128], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3136], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3144], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3152], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3160], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3168], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3176], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3184], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3192], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3200], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3544], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3552], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3560], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3568], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3576], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3584], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3592], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3600], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3608], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3616], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3624], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3632], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3640], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3648], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3656], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3664], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3672], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3680], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3688], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3696], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3704], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4048], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4056], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4064], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4072], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4080], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4088], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4096], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4104], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4112], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4120], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4128], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4136], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4144], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4152], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4160], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4168], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4176], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4184], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4192], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4200], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4208], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "225:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 225b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2704], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2712], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2720], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2728], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2736], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2744], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2752], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2760], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2768], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2776], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2784], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2792], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2800], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2808], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2816], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2824], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2832], rax", + "mov rax, QWORD PTR [rbx+544]", + "mov QWORD PTR [rbx+2840], rax", + "mov rax, QWORD PTR [rbx+576]", + "mov QWORD PTR [rbx+2848], rax", + "mov rax, QWORD PTR [rbx+608]", + "mov QWORD PTR [rbx+2856], rax", + "mov rax, QWORD PTR [rbx+640]", + "mov QWORD PTR [rbx+2864], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3208], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3216], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3224], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3232], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3240], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3248], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3256], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3264], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3272], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3280], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3288], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3296], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3304], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3312], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3320], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3328], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3336], rax", + "mov rax, QWORD PTR [rbx+552]", + "mov QWORD PTR [rbx+3344], rax", + "mov rax, QWORD PTR [rbx+584]", + "mov QWORD PTR [rbx+3352], rax", + "mov rax, QWORD PTR [rbx+616]", + "mov QWORD PTR [rbx+3360], rax", + "mov rax, QWORD PTR [rbx+648]", + "mov QWORD PTR [rbx+3368], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3712], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3720], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3728], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3736], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3744], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3752], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3760], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3768], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3776], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3784], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3792], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3800], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3808], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3816], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3824], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3832], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3840], rax", + "mov rax, QWORD PTR [rbx+560]", + "mov QWORD PTR [rbx+3848], rax", + "mov rax, QWORD PTR [rbx+592]", + "mov QWORD PTR [rbx+3856], rax", + "mov rax, QWORD PTR [rbx+624]", + "mov QWORD PTR [rbx+3864], rax", + "mov rax, QWORD PTR [rbx+656]", + "mov QWORD PTR [rbx+3872], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4216], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4224], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4232], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4240], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4248], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4256], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4264], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4272], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4280], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4288], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4296], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4304], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4312], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4320], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4328], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4336], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4344], rax", + "mov rax, QWORD PTR [rbx+568]", + "mov QWORD PTR [rbx+4352], rax", + "mov rax, QWORD PTR [rbx+600]", + "mov QWORD PTR [rbx+4360], rax", + "mov rax, QWORD PTR [rbx+632]", + "mov QWORD PTR [rbx+4368], rax", + "mov rax, QWORD PTR [rbx+664]", + "mov QWORD PTR [rbx+4376], rax", + "vzeroupper", + "mov rsi, rbx", + "add rsi, 2368", + "mov rbp, r13", + "add rbp, 0", + "mov rdi, QWORD PTR [rbx+4424]", + "mov ecx, 168", + "226:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 227f", + "jmp 228f", + "227:", + "cmp r8d, 8380417", + "jb 229f", + "jmp 230f", + "229:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "230:", + "228:", + "add rsi, 3", + "sub rcx, 1", + "jne 226b", + "mov rax, rdi", + "shr rax, 8", + "and r14d, eax", + "mov rsi, rbx", + "add rsi, 2872", + "mov rbp, r13", + "add rbp, 1024", + "mov rdi, QWORD PTR [rbx+4432]", + "mov ecx, 168", + "231:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 232f", + "jmp 233f", + "232:", + "cmp r8d, 8380417", + "jb 234f", + "jmp 235f", + "234:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "235:", + "233:", + "add rsi, 3", + "sub rcx, 1", + "jne 231b", + "mov rax, rdi", + "shr rax, 8", + "and r14d, eax", + "mov rsi, rbx", + "add rsi, 3376", + "mov rbp, r13", + "add rbp, 2048", + "mov rdi, QWORD PTR [rbx+4440]", + "mov ecx, 168", + "236:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 237f", + "jmp 238f", + "237:", + "cmp r8d, 8380417", + "jb 239f", + "jmp 240f", + "239:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "240:", + "238:", + "add rsi, 3", + "sub rcx, 1", + "jne 236b", + "mov rax, rdi", + "shr rax, 8", + "and r14d, eax", + "mov rsi, rbx", + "add rsi, 3880", + "mov rbp, r13", + "add rbp, 3072", + "mov rdi, QWORD PTR [rbx+4448]", + "mov ecx, 168", + "241:", + "movzx eax, BYTE PTR [rsi]", + "movzx edx, BYTE PTR [rsi+1]", + "movzx r8d, BYTE PTR [rsi+2]", + "and r8d, 127", + "ror r8d, 16", + "ror edx, 24", + "add r8d, edx", + "add r8d, eax", + "cmp rdi, 256", + "jb 242f", + "jmp 243f", + "242:", + "cmp r8d, 8380417", + "jb 244f", + "jmp 245f", + "244:", + "mov DWORD PTR [rbp+rdi*4], r8d", + "add rdi, 1", + "245:", + "243:", + "add rsi, 3", + "sub rcx, 1", + "jne 241b", + "mov rax, rdi", + "shr rax, 8", + "and r14d, eax", + "mov eax, r14d", + "mov r14, QWORD PTR [rbx+4416]", + "mov r13, QWORD PTR [rbx+4408]", + "mov r12, QWORD PTR [rbx+4400]", + "mov rbp, QWORD PTR [rbx+4392]", + "mov rbx, QWORD PTR [rbx+4384]", + "ret", + ) +} + /// `RejBoundedPoly` (FIPS 204 Algorithm 31): writes the polynomial with coefficients in `[-eta, eta]` sampled from the SHAKE256 output of the 66 bytes `*seed` to `*a` (each coefficient modulo `q` = 8380417), and returns 1. Returns 0 if the loop reaches its bound, which is at least 481 bytes of SHAKE256 output (FIPS 204 Appendix C; this happens with probability about 2^-256 or less): `*a` is then unspecified, and the caller must destroy it and treat the operation as failed. /// /// Contract: `VG.Spec.MlDsa.rejBoundedContract`. Not constant time in which half-bytes of the SHAKE256 output it rejects: timing may depend on the pointers, `eta`, and whether each half-byte of the first 1088 bytes of output is rejected (`rejBoundedLeak`), which is independent of the coefficients sampled from those accepted; but not on anything else of the seed or the output. diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs index 90e8d05f4..3ac477362 100644 --- a/src/asm/x86_64/mldsa44.rs +++ b/src/asm/x86_64/mldsa44.rs @@ -2537,7 +2537,7 @@ pub(crate) const VG_MLDSA44_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; /// * `scratch` must be valid for reads and writes of 77824 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). /// * The CPU must support the `avx` and `avx2` target features. #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312], mu: *const [u8; 64], sig: *const [u8; 2420], scratch: *mut [u64; 9728]) -> u32 { @@ -2634,119 +2634,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -2754,270 +2718,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rdi, 4", "sub rcx, 1", "jne 29b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "212:", + "add rdi, 36864", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 212b", + "jne 210b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "216:", + "add rdi, 45056", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", + "jne 211b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "220:", + "add rdi, 53248", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 212b", "mov rdi, r13", "add rdi, 0", "mov esi, 32", @@ -3033,13 +2841,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 213b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -3415,7 +3223,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rdi, 0", "mov ecx, 32", "mov edx, 0", - "222:", + "214:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -3423,7 +3231,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 222b", + "jne 214b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -3440,7 +3248,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, @@ -5980,7 +5788,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: /// * `scratch` must be valid for reads and writes of 77824 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu: *const [u8; 64], sig: *const [u8; 2420], scratch: *mut [u64; 9728]) -> u32 { core::arch::naked_asm!( @@ -6076,119 +5884,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6196,270 +5968,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rdi, 4", "sub rcx, 1", "jne 29b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "212:", + "add rdi, 36864", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 212b", + "jne 210b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "216:", + "add rdi, 45056", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", + "jne 211b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "220:", + "add rdi, 53248", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 212b", "mov rdi, r13", "add rdi, 0", "mov esi, 32", @@ -6475,13 +6091,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 213b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -6857,7 +6473,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rdi, 0", "mov ecx, 32", "mov edx, 0", - "222:", + "214:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -6865,7 +6481,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 222b", + "jne 214b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -6882,7 +6498,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs index 6452825aa..f85fb71ee 100644 --- a/src/asm/x86_64/mldsa65.rs +++ b/src/asm/x86_64/mldsa65.rs @@ -3716,7 +3716,7 @@ pub(crate) const VG_MLDSA65_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; /// * `scratch` must be valid for reads and writes of 103424 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). /// * The CPU must support the `avx` and `avx2` target features. #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952], mu: *const [u8; 64], sig: *const [u8; 3309], scratch: *mut [u64; 12928]) -> u32 { @@ -3826,119 +3826,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -3946,14 +3910,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 29b", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 1", + "mov eax, 0", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -3961,7 +3925,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", + "add rdi, 32768", "mov ecx, 256", "210:", "mov eax, DWORD PTR [rdi]", @@ -3970,23 +3934,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 210b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", + "add rdi, 36864", + "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -3994,14 +3970,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 211b", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 1", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4009,7 +3985,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 38912", + "add rdi, 40960", "mov ecx, 256", "212:", "mov eax, DWORD PTR [rdi]", @@ -4018,23 +3994,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 212b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", + "add rdi, 45056", + "mov ecx, 1024", "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4044,12 +4032,12 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "jne 213b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4057,7 +4045,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 40960", + "add rdi, 49152", "mov ecx, 256", "214:", "mov eax, DWORD PTR [rdi]", @@ -4067,22 +4055,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "sub rcx, 1", "jne 214b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", + "add rdi, 53248", + "mov ecx, 1024", "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4090,14 +4090,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 215b", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4105,7 +4105,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 46080", + "add rdi, 57344", "mov ecx, 256", "216:", "mov eax, DWORD PTR [rdi]", @@ -4114,23 +4114,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", + "add rdi, 61440", + "mov ecx, 1024", "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4138,14 +4150,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 217b", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 65536", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4153,7 +4165,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", + "add rdi, 65536", "mov ecx, 256", "218:", "mov eax, DWORD PTR [rdi]", @@ -4162,23 +4174,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 218b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", + "add rdi, 69632", + "mov ecx, 1024", "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4186,14 +4210,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 219b", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4201,7 +4225,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", + "add rdi, 73728", "mov ecx, 256", "220:", "mov eax, DWORD PTR [rdi]", @@ -4210,22 +4234,20 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 220b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "mov rdi, r13", + "add rdi, 0", + "mov esi, 48", + "mov edx, 49", + "mov rcx, rbx", + "add rcx, 23552", + "mov r8, rbx", + "add r8, 4096", + "call {vg_mldsa_sample_in_ball}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 54272", + "add rdi, 23552", "mov ecx, 256", "221:", "mov eax, DWORD PTR [rdi]", @@ -4234,422 +4256,88 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 221b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 16384", "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "222:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 222b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 18432", "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "223:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 223b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 20480", "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "224:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 224b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "225:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 225b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 29696", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 30720", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 31744", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 32768", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 71680", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 72704", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 73728", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov rdi, r13", - "add rdi, 0", - "mov esi, 48", - "mov edx, 49", - "mov rcx, rbx", - "add rcx, 23552", - "mov r8, rbx", - "add r8, 4096", - "call {vg_mldsa_sample_in_ball}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 23552", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov rdi, rbx", - "add rdi, 16384", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 17408", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 18432", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 19456", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 20480", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 23552", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 32", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", - "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 25600", - "mov rsi, rbx", - "add rsi, 23552", - "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", @@ -5130,7 +4818,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 0", "mov ecx, 48", "mov edx, 0", - "236:", + "222:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -5138,7 +4826,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 236b", + "jne 222b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -5155,6 +4843,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, @@ -8874,7 +8563,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: /// * `scratch` must be valid for reads and writes of 103424 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu: *const [u8; 64], sig: *const [u8; 3309], scratch: *mut [u64; 12928]) -> u32 { core::arch::naked_asm!( @@ -8983,398 +8672,98 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", - "29:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 29b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 36864", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 39936", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 40960", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "mov rdi, rbx", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "28:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 219b", + "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "220:", + "add rdi, 28672", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", - "mov eax, 1", + "jne 29b", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 3", + "mov eax, 0", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9382,47 +8771,59 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 54272", + "add rdi, 32768", "mov ecx, 256", - "221:", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 210b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "222:", + "add rdi, 36864", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", - "mov eax, 3", + "jne 211b", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 3", + "mov eax, 1", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9430,47 +8831,59 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", + "add rdi, 40960", "mov ecx, 256", - "223:", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "224:", + "add rdi, 45056", + "mov ecx, 1024", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "jne 213b", "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9478,47 +8891,59 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 61440", + "add rdi, 49152", "mov ecx, 256", - "225:", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", + "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "226:", + "add rdi, 53248", + "mov ecx, 1024", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 226b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "jne 215b", "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9526,39 +8951,51 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 63488", + "add rdi, 57344", "mov ecx, 256", - "227:", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 227b", + "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "228:", + "add rdi, 61440", + "mov ecx, 1024", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 228b", + "jne 217b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 4", @@ -9576,109 +9013,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 65536", "mov ecx, 256", - "229:", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 229b", + "jne 218b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 69632", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 71680", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "233:", + "add rdi, 69632", + "mov ecx, 1024", + "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 233b", + "jne 219b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 5", @@ -9696,13 +9073,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 73728", "mov ecx, 256", - "234:", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 234b", + "jne 220b", "mov rdi, r13", "add rdi, 0", "mov esi, 48", @@ -9718,13 +9095,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "235:", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 235b", + "jne 221b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -10287,7 +9664,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rdi, 0", "mov ecx, 48", "mov edx, 0", - "236:", + "222:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -10295,7 +9672,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 236b", + "jne 222b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -10312,6 +9689,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs index 79f684dd0..50336abce 100644 --- a/src/asm/x86_64/mldsa87.rs +++ b/src/asm/x86_64/mldsa87.rs @@ -5604,7 +5604,7 @@ pub(crate) const VG_MLDSA87_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; /// * `scratch` must be valid for reads and writes of 144384 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). /// * The CPU must support the `avx` and `avx2` target features. #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592], mu: *const [u8; 64], sig: *const [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { @@ -5740,119 +5740,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5860,23 +5824,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 29b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 31744", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 33792", - "mov ecx, 256", + "add rdi, 31744", + "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5884,23 +5860,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 210b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 34816", - "mov ecx, 256", + "add rdi, 36864", + "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5908,23 +5896,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 211b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 39936", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", + "add rdi, 39936", + "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5932,23 +5932,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", + "add rdi, 45056", + "mov ecx, 1024", "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5956,23 +5968,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 213b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 48128", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", + "add rdi, 48128", + "mov ecx, 1024", "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5980,23 +6004,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", + "add rdi, 53248", + "mov ecx, 1024", "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6004,23 +6040,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 215b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 56320", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", + "add rdi, 56320", + "mov ecx, 1024", "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6028,23 +6076,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 216b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 41984", - "mov ecx, 256", + "add rdi, 61440", + "mov ecx, 1024", "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6052,23 +6112,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 217b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 64512", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 43008", - "mov ecx, 256", + "add rdi, 64512", + "mov ecx, 1024", "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6077,22 +6149,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "sub rcx, 1", "jne 218b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", + "mov rdi, rbx", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", + "add rdi, 69632", + "mov ecx, 1024", "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6100,23 +6184,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 219b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 72704", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", + "add rdi, 72704", + "mov ecx, 1024", "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6124,23 +6220,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 220b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", + "add rdi, 77824", + "mov ecx, 1024", "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6149,22 +6257,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "sub rcx, 1", "jne 221b", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 80896", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", + "add rdi, 80896", + "mov ecx, 1024", "222:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6172,23 +6292,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 222b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 86016", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", + "add rdi, 86016", + "mov ecx, 1024", "223:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6196,23 +6328,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 223b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 89088", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 50176", - "mov ecx, 256", + "add rdi, 89088", + "mov ecx, 1024", "224:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6220,22 +6364,20 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 224b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "mov rdi, r13", + "add rdi, 0", + "mov esi, 64", + "mov edx, 60", + "mov rcx, rbx", + "add rcx, 23552", + "mov r8, rbx", + "add r8, 4096", + "call {vg_mldsa_sample_in_ball}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 51200", + "add rdi, 23552", "mov ecx, 256", "225:", "mov eax, DWORD PTR [rdi]", @@ -6244,971 +6386,473 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 225b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 16384", "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 18432", "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 20480", "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 22528", "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 30720", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 32768", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 34816", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1024", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "236:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 236b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 38912", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "237:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 237b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 66560", + "add rsi, 40960", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 66560", - "mov ecx, 256", - "238:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 238b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 41984", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 67584", + "add rsi, 43008", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 352", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 67584", - "mov ecx, 256", - "239:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 239b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "240:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 240b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "241:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 241b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1152", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "242:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 242b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 47104", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "243:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 243b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 49152", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "244:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 244b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 51200", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 672", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "245:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 245b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "246:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 246b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 77824", - "mov ecx, 256", - "247:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 247b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 10240", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1280", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 78848", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 78848", - "mov ecx, 256", - "248:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 248b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 55296", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 79872", - "mov ecx, 256", - "249:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 249b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 56320", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 80896", - "mov ecx, 256", - "250:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 250b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 57344", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 81920", - "mov ecx, 256", - "251:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 251b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 58368", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 82944", + "add rsi, 59392", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 992", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 82944", - "mov ecx, 256", - "252:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 252b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 83968", - "mov ecx, 256", - "253:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 253b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 86016", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 86016", - "mov ecx, 256", - "254:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 254b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 11264", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1408", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 87040", - "mov ecx, 256", - "255:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 255b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 62464", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 88064", + "add rsi, 63488", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 88064", - "mov ecx, 256", - "256:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 256b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 65536", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 89088", - "mov ecx, 256", - "257:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 257b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 90112", + "add rsi, 67584", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1312", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 90112", - "mov ecx, 256", - "258:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 258b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 91136", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 91136", - "mov ecx, 256", - "259:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 259b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 92160", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 92160", - "mov ecx, 256", - "260:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 260b", - "mov rdi, r13", - "add rdi, 0", - "mov esi, 64", - "mov edx, 60", - "mov rcx, rbx", - "add rcx, 23552", - "mov r8, rbx", - "add r8, 4096", - "call {vg_mldsa_sample_in_ball}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 23552", - "mov ecx, 256", - "261:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 261b", - "mov rdi, rbx", - "add rdi, 16384", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 17408", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 18432", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 19456", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 20480", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 21504", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 22528", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 23552", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 33792", - "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 34816", - "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 32", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", - "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 25600", - "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 23552", "mov rdx, rbx", "add rdx, 24576", "call {vg_mldsa_multiply_ntt_avx2}", @@ -7223,7 +6867,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 8192", + "add rdi, 12288", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7234,60 +6878,60 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1024", + "add rdx, 1536", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 69632", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 70656", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 71680", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 72704", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 74752", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 75776", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbp", - "add rdi, 352", + "add rdi, 1632", "mov rsi, rbx", "add rsi, 24576", "call {vg_mldsa_unpack_t1}", @@ -7314,7 +6958,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 9216", + "add rdi, 13312", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7325,60 +6969,60 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1152", + "add rdx, 1664", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 77824", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 78848", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 79872", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 80896", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 81920", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 82944", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 51200", + "add rsi, 83968", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbp", - "add rdi, 672", + "add rdi, 1952", "mov rsi, rbx", "add rsi, 24576", "call {vg_mldsa_unpack_t1}", @@ -7405,7 +7049,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 10240", + "add rdi, 14336", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7416,60 +7060,60 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1280", + "add rdx, 1792", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 86016", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 87040", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 88064", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 89088", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 90112", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 91136", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 92160", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbp", - "add rdi, 992", + "add rdi, 2272", "mov rsi, rbx", "add rsi, 24576", "call {vg_mldsa_unpack_t1}", @@ -7496,7 +7140,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 11264", + "add rdi, 15360", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7507,618 +7151,854 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1408", + "add rdx, 1920", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 61440", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 62464", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 63488", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 1024", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 64512", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 2048", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", "mov rsi, rbx", - "add rsi, 65536", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rsi, 2048", + "mov rdi, r13", + "add rdi, 0", + "mov ecx, 64", + "mov edx, 0", + "226:", + "movzx eax, BYTE PTR [rsi]", + "movzx r8d, BYTE PTR [rdi]", + "xor rax, r8", + "or rdx, rax", + "add rsi, 1", + "add rdi, 1", + "sub rcx, 1", + "jne 226b", + "sub rdx, 1", + "sbb rax, rax", + "and r15d, eax", + "23:", + "21:", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, + vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, + vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, + vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1, + vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + ) +} + +/// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address. +/// +/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails. +/// +/// # Safety +/// +/// * `seed` must be valid for reads of 32 bytes. +/// * `pk` must be valid for reads and writes of 2592 bytes. +/// * `sk` must be valid for reads and writes of 4896 bytes. +/// * `scratch` must be valid for reads and writes of 144384 bytes. +/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do). +/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk: *mut [u8; 2592], sk: *mut [u8; 4896], scratch: *mut [u64; 18048]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+840], rbx", + "mov QWORD PTR [rcx+848], rbp", + "mov QWORD PTR [rcx+856], r12", + "mov QWORD PTR [rcx+864], r13", + "mov QWORD PTR [rcx+872], r14", + "mov QWORD PTR [rcx+880], r15", + "mov rbx, rcx", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r15d, 1", + "mov eax, 8", + "mov BYTE PTR [rbx+896], al", + "mov eax, 7", + "mov BYTE PTR [rbx+897], al", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 66560", - "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 67584", - "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1312", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, rbx", + "add rcx, 896", + "mov r8d, 2", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 34", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 128", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", - "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rsi, 1024", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1216", "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rsi, 1056", + "mov ecx, 64", + "21:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 21b", + "mov eax, 0", + "mov BYTE PTR [rbx+1281], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 12288", - "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", - "mov rdi, rbx", - "add rdi, 27648", - "mov esi, 15", "mov rdx, rbx", - "add rdx, 1536", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 4096", + "mov ecx, 256", + "22:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 22b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 5120", "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 5120", + "mov ecx, 256", + "23:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 23b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 6144", "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 6144", + "mov ecx, 256", + "24:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 24b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 7168", "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 7168", + "mov ecx, 256", + "25:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 25b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 8192", + "mov ecx, 256", + "26:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 26b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 9216", "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 9216", + "mov ecx, 256", + "27:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 27b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 10240", "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 10240", + "mov ecx, 256", + "28:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 28b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 11264", "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1632", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "add rdi, 11264", + "mov ecx, 256", + "29:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 29b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rdi, 12288", + "mov ecx, 256", + "210:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 210b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 13312", - "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", - "mov rdi, rbx", - "add rdi, 27648", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 1664", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", + "mov ecx, 256", + "211:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 211b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 14336", "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 78848", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 14336", + "mov ecx, 256", + "212:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 212b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 15360", "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 80896", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 15360", + "mov ecx, 256", + "213:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 213b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 16384", "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 82944", - "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 16384", + "mov ecx, 256", + "214:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 214b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 17408", "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1952", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "add rdi, 17408", + "mov ecx, 256", + "215:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 215b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 18432", "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", + "add rdi, 18432", + "mov ecx, 256", + "216:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 216b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 14336", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", - "mov rdi, rbx", - "add rdi, 27648", - "mov esi, 15", + "add rsi, 19456", "mov rdx, rbx", - "add rdx, 1792", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 86016", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdi, 19456", + "mov ecx, 256", + "217:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 217b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 20480", "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 88064", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 20480", + "mov ecx, 256", + "218:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 218b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 21504", "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 90112", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 21504", + "mov ecx, 256", + "219:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 219b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 91136", + "add rsi, 22528", "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 22528", + "mov ecx, 256", + "220:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 220b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 92160", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 2272", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 23552", + "mov ecx, 256", + "221:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 221b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "mov ecx, 256", + "222:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 222b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 25600", "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 25600", + "mov ecx, 256", + "223:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 223b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rsi, 26624", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", + "mov ecx, 256", + "224:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 224b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 15360", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", + "add rsi, 27648", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 27648", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 1920", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", + "mov ecx, 256", + "225:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 225b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, r12", - "add rcx, 0", - "mov r8d, 64", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 64", - "mov rcx, rbx", - "add rcx, 1024", - "mov r8d, 1024", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov ecx, 31", - "mov r8, rbx", - "add r8, 200", - "call {vg_keccak_pad}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbx", - "add rcx, 2048", - "mov r8d, 64", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_squeeze}", - "mov rsi, rbx", - "add rsi, 2048", - "mov rdi, r13", - "add rdi, 0", - "mov ecx, 64", - "mov edx, 0", - "262:", - "movzx eax, BYTE PTR [rsi]", - "movzx r8d, BYTE PTR [rdi]", - "xor rax, r8", - "or rdx, rax", - "add rsi, 1", - "add rdi, 1", - "sub rcx, 1", - "jne 262b", - "sub rdx, 1", - "sbb rax, rax", - "and r15d, eax", - "23:", - "21:", - "mov eax, r15d", - "mov r15, QWORD PTR [rbx+880]", - "mov r14, QWORD PTR [rbx+872]", - "mov r13, QWORD PTR [rbx+864]", - "mov r12, QWORD PTR [rbx+856]", - "mov rbp, QWORD PTR [rbx+848]", - "mov rbx, QWORD PTR [rbx+840]", - "ret", - vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, - vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, - vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, - vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, - vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, - vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, - vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, - vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1, - vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, - vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, - vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint, - vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, - vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, - vg_keccak_pad = sym super::sha3::vg_keccak_pad, - vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, - ) -} - -/// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. -/// -/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. -/// -/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address. -/// -/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails. -/// -/// # Safety -/// -/// * `seed` must be valid for reads of 32 bytes. -/// * `pk` must be valid for reads and writes of 2592 bytes. -/// * `sk` must be valid for reads and writes of 4896 bytes. -/// * `scratch` must be valid for reads and writes of 144384 bytes. -/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before. -/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). -/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do). -/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). -#[unsafe(naked)] -pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk: *mut [u8; 2592], sk: *mut [u8; 4896], scratch: *mut [u64; 18048]) -> u32 { - core::arch::naked_asm!( - "mov QWORD PTR [rcx+840], rbx", - "mov QWORD PTR [rcx+848], rbp", - "mov QWORD PTR [rcx+856], r12", - "mov QWORD PTR [rcx+864], r13", - "mov QWORD PTR [rcx+872], r14", - "mov QWORD PTR [rcx+880], r15", - "mov rbx, rcx", - "mov rbp, rdi", - "mov r12, rsi", - "mov r13, rdx", - "mov r15d, 1", - "mov eax, 8", - "mov BYTE PTR [rbx+896], al", - "mov eax, 7", - "mov BYTE PTR [rbx+897], al", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbp", - "add rcx, 0", - "mov r8d, 32", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 32", - "mov rcx, rbx", - "add rcx, 896", - "mov r8d, 2", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 34", - "mov ecx, 31", - "mov r8, rbx", - "add r8, 200", - "call {vg_keccak_pad}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbx", - "add rcx, 1024", - "mov r8d, 128", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_squeeze}", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "20:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 20b", - "mov rdi, rbx", - "add rdi, 1216", - "mov rsi, rbx", - "add rsi, 1056", - "mov ecx, 64", - "21:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 21b", - "mov eax, 0", - "mov BYTE PTR [rbx+1281], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 28672", + "mov ecx, 256", + "226:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 226b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 4096", + "add rsi, 29696", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8126,23 +8006,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 4096", + "add rdi, 29696", "mov ecx, 256", - "22:", + "227:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 22b", - "mov eax, 1", + "jne 227b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 3", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 5120", + "add rsi, 30720", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8150,23 +8030,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 5120", + "add rdi, 30720", "mov ecx, 256", - "23:", + "228:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 23b", - "mov eax, 2", + "jne 228b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 3", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 6144", + "add rsi, 31744", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8174,23 +8054,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 6144", + "add rdi, 31744", "mov ecx, 256", - "24:", + "229:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 24b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "jne 229b", "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 7168", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8198,23 +8078,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 7168", + "add rdi, 32768", "mov ecx, 256", - "25:", + "230:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 25b", - "mov eax, 4", + "jne 230b", + "mov eax, 1", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8222,23 +8102,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 8192", + "add rdi, 33792", "mov ecx, 256", - "26:", + "231:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 26b", - "mov eax, 5", + "jne 231b", + "mov eax, 2", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 9216", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8246,23 +8126,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 9216", + "add rdi, 34816", "mov ecx, 256", - "27:", + "232:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 27b", - "mov eax, 6", + "jne 232b", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 10240", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8270,23 +8150,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 10240", + "add rdi, 35840", "mov ecx, 256", - "28:", + "233:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 28b", - "mov eax, 0", + "jne 233b", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8294,23 +8174,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 11264", + "add rdi, 36864", "mov ecx, 256", - "29:", + "234:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 29b", - "mov eax, 1", + "jne 234b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 12288", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8318,23 +8198,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 12288", + "add rdi, 37888", "mov ecx, 256", - "210:", + "235:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 210b", - "mov eax, 2", + "jne 235b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 13312", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8342,23 +8222,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 13312", + "add rdi, 38912", "mov ecx, 256", - "211:", + "236:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 211b", - "mov eax, 3", + "jne 236b", + "mov eax, 0", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 14336", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8366,23 +8246,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 14336", + "add rdi, 39936", "mov ecx, 256", - "212:", + "237:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 212b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", + "jne 237b", "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8390,23 +8270,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 15360", + "add rdi, 40960", "mov ecx, 256", - "213:", + "238:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 213b", - "mov eax, 5", + "jne 238b", + "mov eax, 2", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 16384", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8414,23 +8294,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 16384", + "add rdi, 41984", "mov ecx, 256", - "214:", + "239:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 214b", - "mov eax, 6", + "jne 239b", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 17408", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8438,23 +8318,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 17408", + "add rdi, 43008", "mov ecx, 256", - "215:", + "240:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 215b", - "mov eax, 0", + "jne 240b", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 18432", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8462,23 +8342,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 18432", + "add rdi, 44032", "mov ecx, 256", - "216:", + "241:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", - "mov eax, 1", + "jne 241b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8486,23 +8366,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 19456", + "add rdi, 45056", "mov ecx, 256", - "217:", + "242:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 217b", - "mov eax, 2", + "jne 242b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 20480", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8510,23 +8390,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 20480", + "add rdi, 46080", "mov ecx, 256", - "218:", + "243:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 218b", - "mov eax, 3", + "jne 243b", + "mov eax, 0", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 21504", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8534,23 +8414,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 21504", + "add rdi, 47104", "mov ecx, 256", - "219:", + "244:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 219b", - "mov eax, 4", + "jne 244b", + "mov eax, 1", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 22528", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8558,23 +8438,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 22528", + "add rdi, 48128", "mov ecx, 256", - "220:", + "245:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", + "jne 245b", "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8582,23 +8462,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 23552", + "add rdi, 49152", "mov ecx, 256", - "221:", + "246:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", - "mov eax, 6", + "jne 246b", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 24576", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8606,23 +8486,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", + "add rdi, 50176", "mov ecx, 256", - "222:", + "247:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", - "mov eax, 0", + "jne 247b", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 25600", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8630,23 +8510,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 51200", "mov ecx, 256", - "223:", + "248:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", - "mov eax, 1", + "jne 248b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 26624", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8654,23 +8534,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 52224", "mov ecx, 256", - "224:", + "249:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", - "mov eax, 2", + "jne 249b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 27648", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8678,23 +8558,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 27648", + "add rdi, 53248", "mov ecx, 256", - "225:", + "250:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", - "mov eax, 3", + "jne 250b", + "mov eax, 0", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 28672", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8702,23 +8582,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 28672", + "add rdi, 54272", "mov ecx, 256", - "226:", + "251:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 226b", - "mov eax, 4", + "jne 251b", + "mov eax, 1", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 29696", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8726,23 +8606,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 29696", + "add rdi, 55296", "mov ecx, 256", - "227:", + "252:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 227b", - "mov eax, 5", + "jne 252b", + "mov eax, 2", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 30720", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8750,23 +8630,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 30720", + "add rdi, 56320", "mov ecx, 256", - "228:", + "253:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 228b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", + "jne 253b", "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8774,23 +8654,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 31744", + "add rdi, 57344", "mov ecx, 256", - "229:", + "254:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 229b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "jne 254b", "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8798,23 +8678,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 32768", + "add rdi, 58368", "mov ecx, 256", - "230:", + "255:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 230b", - "mov eax, 1", + "jne 255b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 59392", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8822,23 +8702,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 59392", "mov ecx, 256", - "231:", + "256:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 231b", - "mov eax, 2", + "jne 256b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 60416", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8846,1197 +8726,759 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 60416", "mov ecx, 256", - "232:", + "257:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 232b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 257b", + "mov eax, 0", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 35840", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 61440", "mov ecx, 256", - "233:", + "258:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 233b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 258b", + "mov eax, 1", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 36864", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 62464", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 36864", + "add rdi, 62464", "mov ecx, 256", - "234:", + "259:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 234b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 259b", + "mov eax, 2", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 37888", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 63488", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 37888", + "add rdi, 63488", "mov ecx, 256", - "235:", + "260:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 235b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 260b", + "mov eax, 3", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 38912", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 64512", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 38912", + "add rdi, 64512", "mov ecx, 256", - "236:", + "261:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 236b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 261b", + "mov eax, 4", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 39936", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 65536", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 39936", + "add rdi, 65536", "mov ecx, 256", - "237:", + "262:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 237b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "jne 262b", "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 40960", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 66560", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 40960", + "add rdi, 66560", "mov ecx, 256", - "238:", + "263:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 238b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 263b", + "mov eax, 6", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 41984", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 67584", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 41984", + "add rdi, 67584", "mov ecx, 256", - "239:", + "264:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 239b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 264b", + "mov eax, 7", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 43008", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 68608", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 43008", + "add rdi, 68608", "mov ecx, 256", - "240:", + "265:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 240b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 265b", + "mov eax, 8", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 44032", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 69632", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 44032", + "add rdi, 69632", "mov ecx, 256", - "241:", + "266:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 241b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 266b", + "mov eax, 9", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 45056", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 70656", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 45056", + "add rdi, 70656", "mov ecx, 256", - "242:", + "267:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 242b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 267b", + "mov eax, 10", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 46080", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 71680", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 46080", + "add rdi, 71680", "mov ecx, 256", - "243:", + "268:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 243b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 268b", + "mov eax, 11", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 47104", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 72704", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 47104", + "add rdi, 72704", "mov ecx, 256", - "244:", + "269:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 244b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 269b", + "mov eax, 12", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 48128", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 73728", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 48128", + "add rdi, 73728", "mov ecx, 256", - "245:", + "270:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 245b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 270b", + "mov eax, 13", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 49152", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 74752", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 49152", + "add rdi, 74752", "mov ecx, 256", - "246:", + "271:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 246b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 271b", + "mov eax, 14", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 50176", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 75776", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 50176", + "add rdi, 75776", "mov ecx, 256", - "247:", + "272:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 247b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "jne 272b", + "mov rdi, r12", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 51200", - "mov ecx, 256", - "248:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "273:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 248b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "jne 273b", + "mov rdi, r13", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 52224", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 52224", - "mov ecx, 256", - "249:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "274:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 249b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "jne 274b", + "mov rdi, r13", + "add rdi, 32", "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "250:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1120", + "mov ecx, 32", + "275:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 250b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "jne 275b", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 61440", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 128", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "251:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 251b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 62464", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 224", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 63488", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 320", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "252:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 252b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 64512", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 416", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 65536", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 512", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "253:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 253b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 66560", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 608", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 67584", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 704", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "254:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 254b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 68608", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 800", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 58368", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 69632", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 896", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "255:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 255b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 70656", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 992", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 71680", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1088", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "256:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 256b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 72704", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1184", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 60416", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 73728", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1280", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 60416", - "mov ecx, 256", - "257:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 257b", - "mov eax, 0", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 74752", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1376", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", + "add rdi, 75776", "mov esi, 2", - "mov rdx, rbx", - "add rdx, 61440", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1472", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 61440", - "mov ecx, 256", - "258:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 258b", - "mov eax, 1", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 62464", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 62464", - "mov ecx, 256", - "259:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 259b", - "mov eax, 2", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 63488", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 63488", - "mov ecx, 256", - "260:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 260b", - "mov eax, 3", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 64512", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 64512", - "mov ecx, 256", - "261:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 261b", - "mov eax, 4", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 65536", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 65536", - "mov ecx, 256", - "262:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 262b", - "mov eax, 5", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 66560", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 66560", - "mov ecx, 256", - "263:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 263b", - "mov eax, 6", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 67584", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 67584", - "mov ecx, 256", - "264:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 264b", - "mov eax, 7", - "mov BYTE PTR [rbx+1280], al", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 4096", "mov rdx, rbx", - "add rdx, 68608", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", - "add rdi, 68608", - "mov ecx, 256", - "265:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 265b", - "mov eax, 8", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 6144", "mov rdx, rbx", - "add rdx, 69632", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "266:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 266b", - "mov eax, 9", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 7168", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 70656", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "267:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 267b", - "mov eax, 10", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 9216", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 10240", "mov rdx, rbx", - "add rdx, 71680", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "268:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 268b", - "mov eax, 11", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 68608", + "call {vg_mldsa_add}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 72704", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "269:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 269b", - "mov eax, 12", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 32", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 73728", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1568", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "270:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 270b", - "mov eax, 13", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 11264", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 74752", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "271:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 271b", - "mov eax, 14", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 14336", "mov rdx, rbx", - "add rdx, 75776", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "272:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 272b", - "mov rdi, r12", - "add rdi, 0", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "273:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 273b", - "mov rdi, r13", - "add rdi, 0", + "add rsi, 15360", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "274:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 274b", - "mov rdi, r13", - "add rdi, 32", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1120", - "mov ecx, 32", - "275:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 275b", + "add rsi, 17408", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 61440", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 128", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt}", "mov rdi, rbx", - "add rdi, 62464", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 224", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 69632", + "call {vg_mldsa_add}", "mov rdi, rbx", - "add rdi, 63488", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 320", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 64512", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 416", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 352", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 65536", - "mov esi, 2", - "mov edx, 2", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", "mov rcx, r13", - "add rcx, 512", - "mov r8d, 96", + "add rcx, 1984", + "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 66560", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 608", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 67584", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 704", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 68608", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 800", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 69632", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 896", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 70656", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 992", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 71680", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1088", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 72704", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1184", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 73728", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1280", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 74752", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1376", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 75776", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1472", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 61440", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 62464", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 63488", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 64512", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 65536", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 66560", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 67584", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 18432", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 5120", + "add rsi, 19456", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 6144", + "add rsi, 20480", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 7168", + "add rsi, 21504", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 22528", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 9216", + "add rsi, 23552", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 10240", + "add rsi, 24576", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10048,7 +9490,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 68608", + "add rsi, 70656", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10061,7 +9503,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 32", + "add rdx, 672", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10069,55 +9511,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 1568", + "add rcx, 2400", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 25600", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 12288", + "add rsi, 26624", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 13312", + "add rsi, 27648", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 14336", + "add rsi, 28672", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 29696", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 16384", + "add rsi, 30720", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 17408", + "add rsi, 31744", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10129,7 +9571,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 71680", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10142,7 +9584,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 352", + "add rdx, 992", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10150,55 +9592,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 1984", + "add rcx, 2816", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 18432", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 20480", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 21504", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 22528", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 24576", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10210,7 +9652,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 72704", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10223,7 +9665,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 672", + "add rdx, 1312", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10231,55 +9673,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 2400", + "add rcx, 3232", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 25600", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 26624", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 27648", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 28672", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 29696", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 30720", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10291,7 +9733,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 73728", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10304,7 +9746,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 992", + "add rdx, 1632", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10312,55 +9754,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 2816", + "add rcx, 3648", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 35840", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10372,7 +9814,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 74752", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10385,7 +9827,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 1312", + "add rdx, 1952", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10393,55 +9835,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 3232", + "add rcx, 4064", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 44032", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 59392", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 60416", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10453,7 +9895,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 75776", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10466,7 +9908,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 1632", + "add rdx, 2272", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10474,197 +9916,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 3648", + "add rcx, 4480", "mov r8d, 416", "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 50176", - "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 52224", - "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 74752", - "call {vg_mldsa_add}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 1952", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 4064", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 58368", - "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 60416", - "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 75776", - "call {vg_mldsa_add}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 2272", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 4480", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", "add rdi, 0", "mov esi, 136", @@ -13576,7 +12856,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: /// * `scratch` must be valid for reads and writes of 144384 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu: *const [u8; 64], sig: *const [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { core::arch::naked_asm!( @@ -13636,1425 +12916,705 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 524287", "mov ecx, 524288", "mov r8, rbx", - "add r8, 18432", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 18432", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 1984", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 19456", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 19456", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 2624", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 20480", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 20480", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 3264", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 21504", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 21504", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 3904", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 22528", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 22528", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "test r15d, r15d", - "jne 22f", - "jmp 23f", - "22:", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbp", - "add rsi, 0", - "mov ecx, 32", - "24:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", - "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", - "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", - "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", - "29:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 29b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 33792", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 33792", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 34816", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 34816", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 36864", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 39936", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 40960", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 41984", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 41984", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 43008", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 43008", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "220:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 220b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "221:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 221b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "222:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 222b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", - "223:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 223b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 50176", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 50176", - "mov ecx, 256", - "224:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 224b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 51200", - "mov ecx, 256", - "225:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 225b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 58368", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 61440", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 62464", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 63488", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 64512", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "236:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 236b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 65536", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "237:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 237b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 66560", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 66560", - "mov ecx, 256", - "238:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 238b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 67584", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 67584", - "mov ecx, 256", - "239:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 239b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add r8, 18432", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 69632", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdi, 18432", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "mov rdi, r13", + "add rdi, 1984", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 19456", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "240:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 240b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 19456", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 2624", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 20480", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdi, 20480", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "mov rdi, r13", + "add rdi, 3264", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 21504", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "241:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 241b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 21504", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 3904", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 22528", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 71680", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdi, 22528", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "test r15d, r15d", + "jne 22f", + "jmp 23f", + "22:", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "242:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rdi, 896", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 242b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "jne 24b", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 72704", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "25:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 25b", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "243:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "26:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 243b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "jne 26b", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 73728", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "27:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 27b", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "244:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "28:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 244b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "jne 28b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "245:", + "add rdi, 28672", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 245b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", + "jne 29b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 31744", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "246:", + "add rdi, 31744", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 246b", + "jne 210b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 77824", - "mov ecx, 256", - "247:", + "add rdi, 36864", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 247b", + "jne 211b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 78848", + "add rsi, 39936", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 78848", - "mov ecx, 256", - "248:", + "add rdi, 39936", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 248b", + "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 79872", - "mov ecx, 256", - "249:", + "add rdi, 45056", + "mov ecx, 1024", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 249b", + "jne 213b", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 48128", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 80896", - "mov ecx, 256", - "250:", + "add rdi, 48128", + "mov ecx, 1024", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 250b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 81920", - "mov ecx, 256", - "251:", + "add rdi, 53248", + "mov ecx, 1024", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 251b", + "jne 215b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 82944", + "add rsi, 56320", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 82944", - "mov ecx, 256", - "252:", + "add rdi, 56320", + "mov ecx, 1024", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 252b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 83968", - "mov ecx, 256", - "253:", + "add rdi, 61440", + "mov ecx, 1024", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 253b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "jne 217b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 86016", + "add rsi, 64512", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 86016", - "mov ecx, 256", - "254:", + "add rdi, 64512", + "mov ecx, 1024", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 254b", + "jne 218b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 87040", - "mov ecx, 256", - "255:", + "add rdi, 69632", + "mov ecx, 1024", + "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 255b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "jne 219b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 88064", + "add rsi, 72704", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 88064", - "mov ecx, 256", - "256:", + "add rdi, 72704", + "mov ecx, 1024", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 256b", + "jne 220b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 89088", - "mov ecx, 256", - "257:", + "add rdi, 77824", + "mov ecx, 1024", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 257b", + "jne 221b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 90112", + "add rsi, 80896", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 90112", - "mov ecx, 256", - "258:", + "add rdi, 80896", + "mov ecx, 1024", + "222:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 258b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "jne 222b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 91136", + "add rsi, 86016", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 91136", - "mov ecx, 256", - "259:", + "add rdi, 86016", + "mov ecx, 1024", + "223:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 259b", + "jne 223b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 92160", + "add rsi, 89088", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 92160", - "mov ecx, 256", - "260:", + "add rdi, 89088", + "mov ecx, 1024", + "224:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 260b", + "jne 224b", "mov rdi, r13", "add rdi, 0", "mov esi, 64", @@ -15070,13 +13630,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "261:", + "225:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 261b", + "jne 225b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -15915,7 +14475,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rdi, 0", "mov ecx, 64", "mov edx, 0", - "262:", + "226:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -15923,7 +14483,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 262b", + "jne 226b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -15940,7 +14500,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt,