diff --git a/README.md b/README.md index 41decf2f6..df6882e7a 100644 --- a/README.md +++ b/README.md @@ -924,7 +924,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix sampled with four SHAKE128 instances at once +✅ AVX2; SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix and masks sampled with four SHAKE128 or SHAKE256 instances at once ✅ SHA extensions @@ -940,7 +940,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix sampled with four SHAKE128 instances at once +✅ AVX2; SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix and masks sampled with four SHAKE128 or SHAKE256 instances at once ✅ SHA extensions @@ -956,7 +956,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix sampled with four SHAKE128 instances at once +✅ AVX2; SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix and masks sampled with four SHAKE128 or SHAKE256 instances at once ✅ SHA extensions diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml index 03a7cfcff..754ffea29 100644 --- a/docs/algorithms/ml-dsa-44.toml +++ b/docs/algorithms/ml-dsa-44.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa44.rs"] asm = ["mldsa44", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix sampled with four SHAKE128 instances at once" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix and masks sampled with four SHAKE128 or SHAKE256 instances at once" } diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml index c3dbe7821..ab3b16c03 100644 --- a/docs/algorithms/ml-dsa-65.toml +++ b/docs/algorithms/ml-dsa-65.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa65.rs"] asm = ["mldsa65", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix sampled with four SHAKE128 instances at once" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix and masks sampled with four SHAKE128 or SHAKE256 instances at once" } diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml index e960ab15b..3ac4d538e 100644 --- a/docs/algorithms/ml-dsa-87.toml +++ b/docs/algorithms/ml-dsa-87.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa87.rs"] asm = ["mldsa87", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix sampled with four SHAKE128 instances at once" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic, rounding and norm check; matrix and masks sampled with four SHAKE128 or SHAKE256 instances at once" } diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean index 7fa36286d..cf9c555bb 100644 --- a/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean +++ b/lean/VerifiedGarbage/Artifacts/MlDsaSample/X86_64.lean @@ -5,6 +5,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.RejBoundedCT import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.ExpandMask import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.BallCT import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Verified +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.M4Verified /-! # ML-DSA (FIPS 204) on x86-64: the sampling primitives -/ @@ -63,6 +64,28 @@ def artifacts : List Artifact := [ stack := 16 verified := Proof.MlDsa.X86_64.Sample.expandMask_verified spSafe := Code.all_of_allInstrs (by lit_decide) }, + { Spec.MlDsa.expandMask4Api with + target := X86_64.target + doc := Spec.MlDsa.expandMask4Api.doc (notes := ["It calls `vg_mldsa_expand_mask_poly` on each seed, and \ + saves its caller's callee-saved registers in `scratch`."]) + code := Impl.MlDsa.X86_64.Sample.Mask4.expandMask4 + contract := Spec.MlDsa.expandMask4Contract X86_64.abi 24 + stack := 24 + verified := Proof.MlDsa.X86_64.Mask4.expandMask4_verified + spSafe := Code.all_of_allInstrs (by decide +kernel) }, + { Spec.MlDsa.expandMask4Api with + name := Spec.MlDsa.expandMask4Api.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.expandMask4Api.doc (notes := ["It runs the four instances of SHAKE256 at once, in the \ + four 64-bit elements of AVX2 registers (as `vg_mldsa_rej_ntt_poly4_avx2` does with SHAKE128), \ + squeezing five blocks of each, and unpacks the 576 or 640 bytes of each seed's output as \ + `vg_mldsa_expand_mask_poly` does. It saves its caller's callee-saved registers in `scratch`."]) + code := Impl.MlDsa.X86_64.Sample.Mask4.expandMask4Avx2 + contract := Spec.MlDsa.expandMask4Contract X86_64.abi 24 + stack := 24 + verified := Proof.MlDsa.X86_64.Mask4.expandMask4Avx2_verified + spSafe := Code.all_of_allInstrs (by decide +kernel) + features := ["avx", "avx2"] }, { Spec.MlDsa.sampleInBallApi with target := X86_64.target doc := Spec.MlDsa.sampleInBallApi.doc (notes := ["It squeezes 272 bytes of SHAKE256 output (2 blocks) and \ diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean index 46db69523..6c93cbd02 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean @@ -202,6 +202,6 @@ def subAvx2 : Prog isa := def Backend.avx2 : Backend := ⟨nttAvx2, nttInvAvx2, mulAvx2, mulAddAvx2, addAvx2, subAvx2, Round.highBitsAvx2, Round.lowBitsAvx2, Round.normLtAvx2, Round.makeHintAvx2, Round.useHintAvx2, Sample.Rej4.rejNTT4Avx2, - "_avx2"⟩ + Sample.Mask4.expandMask4Avx2, "_avx2"⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean index e823e7fbe..8e3609286 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean @@ -3,6 +3,7 @@ import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Mul import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.AddSub import VerifiedGarbage.Impl.MlDsa.X86_64.Round.Round import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejNtt4 +import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.ExpandMask4 /-! # ML-DSA on x86-64: implementations of the polynomial arithmetic @@ -11,8 +12,8 @@ Key generation, signing and verification call the polynomial arithmetic of one implementation, a `Backend`: the code of `vg_mldsa_ntt`, `vg_mldsa_inv_ntt`, `vg_mldsa_multiply_ntt`, `vg_mldsa_multiply_add_ntt`, `vg_mldsa_add` and `vg_mldsa_sub`, of `vg_mldsa_high_bits`, `vg_mldsa_low_bits`, -`vg_mldsa_norm_lt`, `vg_mldsa_make_hint` and `vg_mldsa_use_hint`, and of `vg_mldsa_rej_ntt_poly4` -(which samples four entries of the matrix `Â` at once), whose names end with `sfx` (e.g. +`vg_mldsa_norm_lt`, `vg_mldsa_make_hint` and `vg_mldsa_use_hint`, and of `vg_mldsa_rej_ntt_poly4` and +`vg_mldsa_expand_mask_poly4` (which sample four polynomials at once), whose names end with `sfx` (e.g. `_avx2`; nothing for the SSE2 code, `sse2`). Each is a variant of the interface `MlDsaArith` on x86-64 (`Variants/MlDsaArith/X86_64/`), and the functions that call them are emitted once for each @@ -37,19 +38,21 @@ structure Backend where makeHint : Prog isa useHint : Prog isa rej4 : Prog isa + expandMask4 : Prog isa /-- What the names of its functions, and of those calling them, end with. -/ sfx : String /-- The SSE2 code. -/ def Backend.sse2 : Backend := ⟨Arith.ntt, Arith.nttInv, Arith.mul, Arith.mulAdd, Arith.add, Arith.sub, Round.highBits, - Round.lowBits, Round.normLt, Round.makeHint, Round.useHint, Sample.Rej4.rejNTT4, ""⟩ + Round.lowBits, Round.normLt, Round.makeHint, Round.useHint, Sample.Rej4.rejNTT4, + Sample.Mask4.expandMask4, ""⟩ /-- Every function empty, which the proofs that the functions calling a backend never write `rsp` (and load MXCSR only to restore it) evaluate in its place. -/ def Backend.empty : Backend := ⟨.block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], - .block [], .block [], .block [], ""⟩ + .block [], .block [], .block [], .block [], ""⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/ExpandMask4.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/ExpandMask4.lean new file mode 100644 index 000000000..fbd04b42b --- /dev/null +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sample/ExpandMask4.lean @@ -0,0 +1,113 @@ +import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.ExpandMask +import VerifiedGarbage.Impl.MlKem.X86_64.Sample4 + +/-! +# ML-DSA on x86-64: `vg_mldsa_expand_mask_poly4` and `vg_mldsa_expand_mask_poly4_avx2` + +`expandMask4(seeds = rdi, gamma1 = esi, a = rdx, scratch = rcx)` runs +`ExpandMask`'s sampling of a polynomial (`vg_mldsa_expand_mask_poly`) on four +66-byte seeds, to the four polynomials from `a`. Both keep `scratch` in +`rbx`, `seeds` in `r12`, `a` in `r13` and `γ₁` in `r14`, and save their +caller's values and `rbp`'s in `scratch[5088..5128)`. The baseline +implementation (`expandMask4`) calls `vg_mldsa_expand_mask_poly` on each +seed, with its scratch space from byte 6144 of `scratch`. + +The one for AVX2 (`expandMask4Avx2`) runs the four SHAKE256 instances at +once in the four 64-bit elements of `ymm` registers, as +`vg_mlkem_sample_ntt4_avx2` runs four of SHAKE128 (`Impl/MlKem/X86_64/Sample4.lean`, +whose layout of the first 2368 bytes of `scratch` it shares: the four +states, the second buffer of the permutation and the table of the round +constants). Each seed is 66 bytes, so the padded message is one block of +136 bytes: the code zeroes the states, writes the seed's bytes, SHAKE's +suffix `0x1f` (at byte 66) and the last bit of the padding (`0x80`, at byte +135) into each, and permutes them; it then copies the first 136 bytes of +each state to its output (from byte `2368 + 680 k`), and permutes them +again, five times in all (680 bytes, which hold the 640 that `γ₁ = 2¹⁹` +needs and the 576 of `γ₁ = 2¹⁷`). It then unpacks the output of each seed +with the loop of `vg_mldsa_expand_mask_poly` (`emBody`, for `c = 18` or +`20`, chosen by a branch on the public `γ₁`). + +There is no branch on data, and every address depends only on the pointers +and `γ₁`: it is constant time. +-/ + +namespace VG.Impl.MlDsa.X86_64.Sample.Mask4 + +open VG.X86_64 +open VG.Impl.MlKem.X86_64 (at_) +open VG.Impl.MlKem.X86_64.Sample4 (zero4 permArgs oRc oBuf) +open VG.Impl.Sha3.X86_64.X4 (permute4 rcTable) + +/-- Where the callee-saved registers are saved. -/ +def oSave : Nat := 5088 + +/-- The scratch space of `vg_mldsa_expand_mask_poly`, in the baseline implementation. -/ +def oScalar : Nat := 6144 + +/-- The registers saved, at `scratch + oSave + 8 k`. -/ +def saved : List Reg := [.rbx, .rbp, .r12, .r13, .r14] + +/-- Save the callee-saved registers, and keep the pointers and `γ₁`. -/ +def pro : List Instr := + (List.range 5).map (fun k => .store (at_ .rcx (oSave + 8 * k)) (saved.getD k .rbx)) ++ + [.mov .rbx (.reg .rcx), .mov .r12 (.reg .rdi), .mov .r13 (.reg .rdx), .mov .r14 (.reg .rsi)] + +/-- Restore the callee-saved registers (`rbx` last). -/ +def epi : List Instr := + ((List.range 4).map fun k => .mov (saved.getD (4 - k) .rbx) (.mem (at_ .rbx (oSave + 8 * (4 - k))))) ++ + [.mov .rbx (.mem (at_ .rbx oSave))] + +/-- Bytes 0 to 65 of state `k`: the 66 bytes of seed `k`, as eight lanes and +two bytes. -/ +def seedLanes (k : Nat) : List Instr := + (List.range 8).flatMap (fun i => + [.mov .rax (.mem (at_ .r12 (66 * k + 8 * i))), .store (at_ .rbx (32 * i + 8 * k)) .rax]) ++ + [.movzx8 .rax (at_ .r12 (66 * k + 64)), .store8 (at_ .rbx (256 + 8 * k)) .rax, + .movzx8 .rax (at_ .r12 (66 * k + 65)), .store8 (at_ .rbx (256 + 8 * k + 1)) .rax] + +/-- The padded blocks of the four seeds, XORed into the zero states: the +seeds, SHAKE's suffix at byte 66 (byte 2 of lane 8) and `0x80` at byte 135 +(byte 7 of lane 16). -/ +def absorb4 : List Instr := + zero4 ++ (List.range 4).flatMap seedLanes ++ + .mov32 .rax (.imm 0x1f) :: (List.range 4).flatMap (fun k => [.store8 (at_ .rbx (256 + 8 * k + 2)) .rax]) ++ + .mov32 .rax (.imm 0x80) :: (List.range 4).flatMap fun k => [.store8 (at_ .rbx (512 + 8 * k + 7)) .rax] + +/-- The first 136 bytes of each state to block `b` of its output. -/ +def extract (b : Nat) : List Instr := + (List.range 4).flatMap fun k => (List.range 17).flatMap fun i => + [.mov .rax (.mem (at_ .rbx (32 * i + 8 * k))), .store (at_ .rbx (oBuf + 680 * k + 136 * b + 8 * i)) .rax] + +/-- Permute the states and squeeze block `b`. -/ +def squeeze4 (b : Nat) : Prog isa := + .seq (.block permArgs) (.seq permute4 (.block (extract b))) + +/-- The coefficients of polynomial `k` from the output of seed `k`, `c` bits each. -/ +def unpack (c k : Nat) : Prog isa := + .seq (.block [.mov .rsi (.reg .rbx), .alu .add .rsi (.imm (BitVec.ofNat 32 (oBuf + 680 * k))), + .mov .rdi (.reg .r13), .alu .add .rdi (.imm (BitVec.ofNat 32 (1024 * k))), .mov32 .rcx (.imm 64)]) + (.loop (.block (emBody c)) .ne) + +def unpack4 (c : Nat) : Prog isa := .seq (unpack c 0) (.seq (unpack c 1) (.seq (unpack c 2) (unpack c 3))) + +/-- `vg_mldsa_expand_mask_poly4_avx2`. `vzeroupper` clears the upper halves +of the vector registers after the last permutation. -/ +def expandMask4Avx2 : Prog isa := + .seq (.block (pro ++ rcTable .rbx (oRc / 32) ++ absorb4)) + (.seq (squeeze4 0) (.seq (squeeze4 1) (.seq (squeeze4 2) (.seq (squeeze4 3) (.seq (squeeze4 4) + (.seq (.block [.vop .vzeroupper, .alu32 .cmp .r14 (.imm 0x20000)]) + (.seq (.ite .e (unpack4 18) (unpack4 20)) (.block epi)))))))) + +/-- `vg_mldsa_expand_mask_poly` on seed `k`. -/ +def callK (k : Nat) : Prog isa := + .seq (.block [.mov .rdi (.reg .r12), .alu .add .rdi (.imm (BitVec.ofNat 32 (66 * k))), .mov .rsi (.reg .r14), + .mov .rdx (.reg .r13), .alu .add .rdx (.imm (BitVec.ofNat 32 (1024 * k))), .mov .rcx (.reg .rbx), + .alu .add .rcx (.imm (BitVec.ofNat 32 oScalar))]) + (.call "vg_mldsa_expand_mask_poly" expandMask) + +/-- `vg_mldsa_expand_mask_poly4`: `vg_mldsa_expand_mask_poly` on each seed, +with the prologue and epilogue of `vg_mldsa_expand_mask_poly4_avx2`. -/ +def expandMask4 : Prog isa := + .seq (.block pro) (.seq (callK 0) (.seq (callK 1) (.seq (callK 2) (.seq (callK 3) (.block epi))))) + +end VG.Impl.MlDsa.X86_64.Sample.Mask4 diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean index 066d7b621..9e301eb71 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean @@ -46,6 +46,8 @@ structure Prims where hintBitPack : Prog isa /-- `vg_mldsa_rej_ntt_poly4` -/ rej4 : Prog isa + /-- `vg_mldsa_expand_mask_poly4` -/ + expandMask4 : Prog isa /-- What the names of the polynomial arithmetic's functions end with (`Arith.Backend`). -/ sfx : String := "" @@ -57,7 +59,8 @@ the iterations left (`CNT`), the counter `κ` (`KAP`) and the number of 1s of the hint (`ONES`) at 888, 896 and 904 (8 bytes each); the seed of `RejNTTPoly` (`RS`, 34 bytes) at 912; the seed of `ExpandMask` (`MS`, `ρ″` and two bytes) at 960; `c̃` (`CT`, up to 64 bytes) at 1040; the four -seeds of `vg_mldsa_rej_ntt_poly4` (`RS4`, 136 bytes) at 1152; the +seeds of `vg_mldsa_rej_ntt_poly4` (`RS4`, 136 bytes) at 1152; the four +seeds of `vg_mldsa_expand_mask_poly4` (`MS4`, 264 bytes) at 1296; the encoding of `w₁` (`W1`, up to 1024 bytes) at 2048; the working space of the primitives (`PS`, 2048 bytes) at 3072; and polynomials of 1024 bytes from 5120 (`P i`). -/ @@ -70,6 +73,7 @@ def oRS : Nat := 912 def oMS : Nat := 960 def oCT : Nat := 1040 def oRS4 : Nat := 1152 +def oMS4 : Nat := 1296 def oW1 : Nat := 2048 def oPS : Nat := 3072 /-- Polynomial `i`. -/ @@ -194,6 +198,11 @@ def rej4At (a w : Ptr) : Prog isa := .seq (callP ("vg_mldsa_rej_ntt_poly4" ++ P.sfx) P.rej4 [.ptr (sc oRS4), .ptr a, .ptr w]) (.block [.alu32 .and .r15 (.reg .rax)]) +/-- Four polynomials of `ExpandMask` from the four seeds at `MS4` to the four polynomials from `a`, +with the working space `w`. -/ +def mask4At (gamma1 : Nat) (a w : Ptr) : Prog isa := + callP ("vg_mldsa_expand_mask_poly4" ++ P.sfx) P.expandMask4 [.ptr (sc oMS4), .imm gamma1, .ptr a, .ptr w] + /-- A polynomial of `ExpandMask` from the seed at `MS` to `a`. -/ def maskAt (gamma1 : Nat) (a : Ptr) : Prog isa := callP "vg_mldsa_expand_mask_poly" P.expandMask [.ptr (sc oMS), .imm gamma1, .ptr a, .ptr (sc oPS)] diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Sign.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Sign.lean index 3f09192a2..f8c976561 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Sign.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Sign.lean @@ -21,7 +21,9 @@ keeps `scratch` in `rbx`, `sk` in `rbp`, `mu` in `r12`, `rnd` in `r13` and 2. `ŝ₁`, `ŝ₂` and `t̂₀`: the `NTT` of the `BitUnpack` of their pieces of `sk`; and `ρ″ = H(K ‖ rnd ‖ μ, 64)` to `MS`. 3. The rejection sampling loop, at most 814 iterations (`minBounds.sign`), - with the counter `κ` at `KAP`: the commitment (`y ← ExpandMask(ρ″, κ)`, + with the counter `κ` at `KAP`: the commitment (`y ← ExpandMask(ρ″, κ)`: + four polynomials at a time (`vg_mldsa_expand_mask_poly4`, from the seeds + `ρ″ ‖ κ + r` at `MS4`), then the last `ℓ mod 4` one at a time; `ŷ = NTT(y)`, `w = NTT⁻¹(Â ŷ)`, and `c̃ = H(μ ‖ w1Encode(HighBits(w)), λ/4)` at `CT`), then `c = SampleInBall(c̃)` and, if it succeeded, every validity check of the iteration, combined without a branch into `r15`: the norms @@ -144,15 +146,29 @@ def decode : Prog isa := /-! ### An iteration -/ +/-- The two bytes of `κ + r` to `scratch + o`. -/ +def setKap (o r : Nat) : List Instr := + [.mov .rax (.mem (at_ .rbx oKAP)), .alu .add .rax (.imm (BitVec.ofNat 32 r)), .store8 (at_ .rbx o) .rax, + .shift .shr .rax 8, .store8 (at_ .rbx (o + 1)) .rax] + /-- The two bytes of `κ + r` to `MS + 64`. -/ -def setKappa (r : Nat) : List Instr := - [.mov .rax (.mem (at_ .rbx oKAP)), .alu .add .rax (.imm (BitVec.ofNat 32 r)), .store8 (at_ .rbx (oMS + 64)) .rax, - .shift .shr .rax 8, .store8 (at_ .rbx (oMS + 65)) .rax] +def setKappa (r : Nat) : List Instr := setKap (oMS + 64) r /-- `y[r]` and `ŷ[r] = NTT(y[r])`. -/ def maskR (r : Nat) : Prog isa := .seq (.block (setKappa r)) (.seq (maskAt P p.γ₁ (yP p r)) (.seq (copy (yhP p r) (yP p r) 1024) (nttAt P (yhP p r)))) +/-- `ŷ[r] = NTT(y[r])`. -/ +def yhR (r : Nat) : Prog isa := .seq (copy (yhP p r) (yP p r) 1024) (nttAt P (yhP p r)) + +/-- `ρ″ ‖ κ + 4g + k` to seed `k` of `MS4`. -/ +def cpM4 (g k : Nat) : Prog isa := + .seq (copy (sc (oMS4 + 66 * k)) (sc oMS) 64) (.block (setKap (oMS4 + 66 * k + 64) (4 * g + k))) + +/-- `y[4g], …, y[4g + 3]`, four at a time, and their `ŷ`. -/ +def mask4 (g : Nat) : Prog isa := + .seq (seqR (cpM4 g) 0 4) (.seq (mask4At P p.γ₁ (yP p (4 * g)) (r4P p)) (seqR (yhR P p) (4 * g) 4)) + /-- `w[i] = NTT⁻¹(∑_j Â[i, j] ŷ[j])`. -/ def rowW (i : Nat) : Prog isa := .seq (mulAt P (wP p i) (aP p i 0) (yhP p 0)) @@ -164,8 +180,9 @@ def w1R (i : Nat) : Prog isa := /-- `y`, `ŷ`, `w`, `w₁` and `c̃ = H(μ ‖ w1Encode(w₁), λ/4)` to `CT`. -/ def commit : Prog isa := - .seq (seqR (maskR P p) 0 p.ℓ) (.seq (seqR (rowW P p) 0 p.k) (.seq (seqR (w1R P p) 0 p.k) - (shakeAt [((.r12, 0), 64), (sc oW1, p.k * w1Len p)] (sc oCT) (cLen p)))) + .seq (seqR (mask4 P p) 0 (p.ℓ / 4)) (.seq (seqR (maskR P p) (4 * (p.ℓ / 4)) (p.ℓ % 4)) + (.seq (seqR (rowW P p) 0 p.k) (.seq (seqR (w1R P p) 0 p.k) + (shakeAt [((.r12, 0), 64), (sc oW1, p.k * w1Len p)] (sc oCT) (cLen p))))) /-- `z[r] = y[r] + NTT⁻¹(ĉ ŝ₁[r])` (in `y[r]`), and its norm. -/ def zR (r : Nat) : Prog isa := diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean index 4e38a0278..67ee23e8a 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean @@ -26,8 +26,9 @@ of `vg_mldsa_rej_ntt_poly4` (`SB4`, 136 bytes) at 2560; the working space of the primitives at 4096 (2048 bytes); and polynomials of 1024 bytes from 8192 (`P j`): the hint `h` (polynomials 0 to 7, of which the first `k`), `z` (8 to 14), `c` (15), two temporaries (16, 17), `w′` (18), `w′₁` -(19), and `Â[r, s]` (`20 + 8r + s`), then the working space of -`vg_mldsa_rej_ntt_poly4` (8 KiB, after the last row of `Â`). +(19), and `Â[r, s]` (`20 + ℓr + s`, entry `ℓr + s` of `Â` in order), then the +working space of `vg_mldsa_rej_ntt_poly4` (8 KiB, from polynomial `20 + 8k`, after +`Â` for every `ℓ` ≤ 8). -/ namespace VG.Impl.MlDsa.X86_64.Verify @@ -62,7 +63,8 @@ abbrev pT : Ptr := pS 16 abbrev pT2 : Ptr := pS 17 abbrev pW : Ptr := pS 18 abbrev pW1 : Ptr := pS 19 -abbrev pA (r s : Nat) : Ptr := pS (20 + 8 * r + s) +/-- `Â[r, s]`, for rows of `l` entries. -/ +abbrev pA (l r s : Nat) : Ptr := pS (20 + l * r + s) /-! ## Moves -/ diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean index 4ef3f67af..9f4ade74a 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Verify.lean @@ -17,10 +17,9 @@ in `scratch`. 2. `z[i] = BitUnpack` of the `i`-th piece of `σ` (polynomial `8 + i`), and `r15 ← r15 ∧ (‖z[i]‖∞ < γ₁ - β)`; it returns 0 if one of them is not. 3. `ρ` (`pk[0 : 32]`) to `SB` and to each seed of `SB4`, and - `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)` (polynomial `20 + 8r + s`), four - entries of a row at a time (`vg_mldsa_rej_ntt_poly4`): those from 0, then - for `ℓ = 7` those from 3 (sampling entry 3 again), or for `ℓ = 5` entry 4 - alone; `c = SampleInBall(c̃)` (polynomial 15). Each + `Â[r, s] = RejNTTPoly(ρ ‖ s ‖ r)` (entry `e = ℓr + s`, polynomial + `20 + e`), four entries at a time (`vg_mldsa_rej_ntt_poly4`), then the + last `kℓ mod 4` one at a time; `c = SampleInBall(c̃)` (polynomial 15). Each sampler's result is ANDed into `r15`, and its output masked with it (`sampled`): a sampler that fails leaves its output unspecified, and masking makes it reduced (zero) without a branch on the result, which @@ -71,25 +70,22 @@ def zOne (i : Nat) : Prog isa := .seq (bitUnpackAt P (.r13, p.ctildeLen + lenZ p * i) (lenZ p) (p.γ₁ - 1) p.γ₁ (pZ i)) (.seq (normLtAt P (pZ i) (p.γ₁ - p.β)) (.block and15)) -/-- `Â[r, s]`, entry `e = 8r + s`. -/ +/-- Entry `e` of `Â`, `Â[e / ℓ, e mod ℓ]`. -/ def aOne (e : Nat) : Prog isa := - .seq (.block (setB (sc (oSB + 32)) (e % 8) ++ setB (sc (oSB + 33)) (e / 8))) + .seq (.block (setB (sc (oSB + 32)) (e % p.ℓ) ++ setB (sc (oSB + 33)) (e / p.ℓ))) (sampled (rejNttAt P (pS (20 + e))) (pS (20 + e))) /-- Where `vg_mldsa_rej_ntt_poly4` works: after the last row of `Â`. -/ def oR4 : Nat := oP (20 + 8 * p.k) -/-- The bytes `s ‖ r` of seed `k` of `SB4`. -/ -def setSR (r s k : Nat) : List Instr := setB (sc (oSB4 + 34 * k + 32)) (s + k) ++ setB (sc (oSB4 + 34 * k + 33)) r +/-- The bytes `s ‖ r` of entry `e + k` to seed `k` of `SB4`. -/ +def setSR (e k : Nat) : List Instr := + setB (sc (oSB4 + 34 * k + 32)) ((e + k) % p.ℓ) ++ setB (sc (oSB4 + 34 * k + 33)) ((e + k) / p.ℓ) -/-- `Â[r, s], …, Â[r, s + 3]`. -/ -def aGrp (r s : Nat) : Prog isa := - .seq (.block (setSR r s 0)) (.seq (.block (setSR r s 1)) (.seq (.block (setSR r s 2)) (.seq (.block (setSR r s 3)) - (sampled4 (rej4At P (pA r s) (sc (oR4 p))) (pA r s))))) - -/-- The entries of row `r` of `Â`: four from 0, then the last four if `ℓ = 7`, or the last one if `ℓ = 5`. -/ -def aRow (r : Nat) : Prog isa := - .seq (aGrp P p r 0) (if p.ℓ = 7 then aGrp P p r 3 else seqR (aOne P) (8 * r + 4) (p.ℓ - 4)) +/-- Entries `4g, …, 4g + 3` of `Â`. -/ +def aGrp (g : Nat) : Prog isa := + .seq (.block (setSR p (4 * g) 0)) (.seq (.block (setSR p (4 * g) 1)) (.seq (.block (setSR p (4 * g) 2)) + (.seq (.block (setSR p (4 * g) 3)) (sampled4 (rej4At P (pS (20 + 4 * g)) (sc (oR4 p))) (pS (20 + 4 * g)))))) /-- `ρ` to `SB` and to the four seeds of `SB4`. -/ def rhos : Prog isa := @@ -98,12 +94,12 @@ def rhos : Prog isa := /-- `ρ` to `SB` and `SB4`, `Â`, and `c`. -/ def samples : Prog isa := - .seq rhos (.seq (seqR (aRow P p) 0 p.k) - (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC)) + .seq rhos (.seq (seqR (aGrp P p) 0 (p.k * p.ℓ / 4)) (.seq (seqR (aOne P p) (4 * (p.k * p.ℓ / 4)) (p.k * p.ℓ % 4)) + (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC))) /-- `Σₛ Â[r, s] ẑ[s]` to `W`. -/ def dot (r : Nat) : Prog isa := - .seq (mulAt P pW (pA r 0) (pZ 0)) (seqR (fun s => mulAddAt P pW (pA r s) (pZ s)) 1 (p.ℓ - 1)) + .seq (mulAt P pW (pA p.ℓ r 0) (pZ 0)) (seqR (fun s => mulAddAt P pW (pA p.ℓ r s) (pZ s)) 1 (p.ℓ - 1)) /-- Row `r` of `w′`, `w′₁`, packed to `B`. -/ def row (r : Nat) : Prog isa := diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean index ea96daadc..65d90d573 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean @@ -8,6 +8,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Round.Bits import VerifiedGarbage.Proof.MlDsa.X86_64.Round.MakeHint import VerifiedGarbage.Proof.MlDsa.X86_64.Round.NormLt import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.Rej4Verified +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.M4Verified /-! # ML-DSA on x86-64: what the callers of the polynomial arithmetic need of it @@ -47,6 +48,15 @@ structure Rej4Ok (c : Prog isa) : Prop where ret : ∀ s t s', (Spec.MlDsa.rejNTT4Contract X86_64.abi 24).pre s → Exec isa c s t s' → (s'.gpr .rax).setWidth 32 = Rej4.rej4Res s.mem (s.gpr .rdi) +/-- What a caller needs of `vg_mldsa_expand_mask_poly4`, which calls three +deep and takes 24 bytes of stack. -/ +structure Mask4Ok (c : Prog isa) : Prop where + ver : Verified X86_64.target c (Spec.MlDsa.expandMask4Contract X86_64.abi 24) + nosp : NoSp c + depth : c.depth ≤ 3 + ctl : ctlOk c = true + sp : c.all (fun i => !isa.writesSp i) = true + /-- Each function of the backend `B` meets its contract, and is safe to call. -/ structure BackendOk (B : Backend) : Prop where ntt : FnOk (fun S => Spec.MlDsa.nttContract X86_64.abi S) B.ntt @@ -61,6 +71,7 @@ structure BackendOk (B : Backend) : Prop where makeHint : FnOk (fun S => Spec.MlDsa.makeHintContract X86_64.abi S) B.makeHint useHint : FnOk (fun S => Spec.MlDsa.useHintContract X86_64.abi S) B.useHint rej4 : Rej4Ok B.rej4 + expandMask4 : Mask4Ok B.expandMask4 /-- An implementation of the polynomial arithmetic on x86-64. -/ structure ArithImpl where @@ -102,7 +113,9 @@ def ArithImpl.sse2 : ArithImpl where useHint := FnOk.of Round.useHint_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) rej4 := ⟨Rej4.rejNTT4_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), by decide +kernel, - by decide +kernel, Code.all_of_allInstrs (by decide +kernel), fun _ _ _ => Rej4.rejNTT4_ret⟩ } + by decide +kernel, Code.all_of_allInstrs (by decide +kernel), fun _ _ _ => Rej4.rejNTT4_ret⟩ + expandMask4 := ⟨Mask4.expandMask4_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), by decide +kernel, + by decide +kernel, Code.all_of_allInstrs (by decide +kernel)⟩ } features := [] end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean index 275770ff0..19bdd3a3b 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean @@ -43,7 +43,9 @@ def ArithImpl.avx2 : ArithImpl where useHint := FnOk.of Round.useHintY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) rej4 := ⟨Rej4.rejNTT4Avx2_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), by decide +kernel, - by decide +kernel, Code.all_of_allInstrs (by decide +kernel), fun _ _ _ => Rej4.rejNTT4Avx2_ret⟩ } + by decide +kernel, Code.all_of_allInstrs (by decide +kernel), fun _ _ _ => Rej4.rejNTT4Avx2_ret⟩ + expandMask4 := ⟨Mask4.expandMask4Avx2_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), + by decide +kernel, by decide +kernel, Code.all_of_allInstrs (by decide +kernel)⟩ } features := ["avx", "avx2"] end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/SignFn.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/SignFn.lean index f0017ff15..119cd5e2c 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/SignFn.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/SignFn.lean @@ -57,12 +57,12 @@ theorem signFn (v : ArithImpl) {p : Params} (hp : p ∈ params) : refine ⟨sign_correct (prims_okWith v) h3 (Proof.MlDsa.X86_64.Sign.sign_ctl v h3), sign_ct (prims_okWith v) h3, noSp_of (Same.ok (sign_same (Comp.all _) (noSpB_of v.ok.ntt.nosp) (noSpB_of v.ok.invNtt.nosp) (noSpB_of v.ok.mul.nosp) (noSpB_of v.ok.mulAdd.nosp) (noSpB_of v.ok.add.nosp) (noSpB_of v.ok.sub.nosp) - (noSpB_of v.ok.rej4.nosp) (noSpB_of v.ok.highBits.nosp) (noSpB_of v.ok.lowBits.nosp) - (noSpB_of v.ok.normLt.nosp) (noSpB_of v.ok.makeHint.nosp) p) (sign0_noSp h3)), + (noSpB_of v.ok.rej4.nosp) (noSpB_of v.ok.expandMask4.nosp) (noSpB_of v.ok.highBits.nosp) + (noSpB_of v.ok.lowBits.nosp) (noSpB_of v.ok.normLt.nosp) (noSpB_of v.ok.makeHint.nosp) p) (sign0_noSp h3)), of_decide_eq_true (Same.ok (sign_same (depthCompN 3) (dep2 v.ok.ntt.depth) (dep2 v.ok.invNtt.depth) (dep2 v.ok.mul.depth) (dep2 v.ok.mulAdd.depth) (dep2 v.ok.add.depth) (dep2 v.ok.sub.depth) - (dep v.ok.rej4.depth) (dep2 v.ok.highBits.depth) (dep2 v.ok.lowBits.depth) (dep2 v.ok.normLt.depth) - (dep2 v.ok.makeHint.depth) p) (sign0_depth h3))⟩ + (dep v.ok.rej4.depth) (dep v.ok.expandMask4.depth) (dep2 v.ok.highBits.depth) (dep2 v.ok.lowBits.depth) + (dep2 v.ok.normLt.depth) (dep2 v.ok.makeHint.depth) p) (sign0_depth h3))⟩ theorem kabs_spSafe : Impl.Sha3.X86_64.Stream.absorb.all (fun i => !isa.writesSp i) = true := by decide +kernel theorem kpad_spSafe : Impl.Sha3.X86_64.Stream.pad.all (fun i => !isa.writesSp i) = true := by decide +kernel diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/VerifyFn.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/VerifyFn.lean index 1ae2b3ce4..141d0b694 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/VerifyFn.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/VerifyFn.lean @@ -18,7 +18,7 @@ namespace VG.Proof.MlDsa.X86_64.Message open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Message open VG.Proof.MlDsa.Message open VG.Proof.MlDsa.X86_64 (Comp Same Same.ok ArithImpl) -open VG.Impl.MlDsa.X86_64.Verify (Prims callAt seqR ifOk sampled hint zOne aOne aGrp aRow samples dot row compute) +open VG.Impl.MlDsa.X86_64.Verify (Prims callAt seqR ifOk sampled hint zOne aOne aGrp samples dot row compute) open VG.Proof.MlDsa.X86_64.Verify (P0 PrimsOk primsWith prims_okWith verify_correct verify_ct) open VG.Spec.MlDsa @@ -58,7 +58,7 @@ variable {P : Prims} (hP : PrimsM mc P) (p : Params) include hP theorem verify_same : Same m (Impl.MlDsa.X86_64.Verify.verify P p) (Impl.MlDsa.X86_64.Verify.verify P0 p) := by - have aOne : ∀ e, Same m (aOne P e) (aOne P0 e) := fun e => + have aOne : ∀ e, Same m (aOne P p e) (aOne P0 p e) := fun _ => Same.seq hm rfl (Same.sampled hm (Same.callAt hm hP.rejNtt _ _ _) _) have dot : ∀ r, Same m (dot P p r) (dot P0 p r) := fun r => Same.seq hm (Same.callAt hm hP.mul _ _ _) (Same.seqRV hm (fun _ => Same.callAt hm hP.mulAdd _ _ _) _ _) @@ -67,17 +67,12 @@ theorem verify_same : Same m (Impl.MlDsa.X86_64.Verify.verify P p) (Impl.MlDsa.X (Same.seq hm (Same.callAt hm hP.mul _ _ _) (Same.seq hm (Same.callAt hm hP.sub _ _ _) (Same.seq hm (Same.callAt hm hP.invNtt _ _ _) (Same.seq hm (Same.callAt hm hP.useHint _ _ _) (Same.callAt hm hP.simpleBitPack _ _ _))))))) - have aGrp : ∀ r s, Same m (aGrp P p r s) (aGrp P0 p r s) := fun _ _ => + have aGrp : ∀ g, Same m (aGrp P p g) (aGrp P0 p g) := fun _ => Same.seq hm rfl (Same.seq hm rfl (Same.seq hm rfl (Same.seq hm rfl (Same.seq hm (Same.callAt hm hP.rej4 _ _ _) rfl)))) - have aRow : ∀ r, Same m (aRow P p r) (aRow P0 p r) := fun r => by - unfold Impl.MlDsa.X86_64.Verify.aRow - split - · exact Same.seq hm (aGrp r 0) (aGrp r 3) - · exact Same.seq hm (aGrp r 0) (Same.seqRV hm aOne _ _) have samples : Same m (samples P p) (samples P0 p) := - Same.seq hm rfl (Same.seq hm (Same.seqRV hm aRow _ _) - (Same.sampled hm (Same.callAt hm hP.ball _ _ _) _)) + Same.seq hm rfl (Same.seq hm (Same.seqRV hm aGrp _ _) (Same.seq hm (Same.seqRV hm aOne _ _) + (Same.sampled hm (Same.callAt hm hP.ball _ _ _) _))) have compute : Same m (compute P p) (compute P0 p) := Same.seq hm (Same.seqRV hm (fun _ => Same.callAt hm hP.ntt _ _ _) _ _) (Same.seq hm (Same.callAt hm hP.ntt _ _ _) (Same.seq hm (Same.seqRV hm row _ _) rfl)) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/ExpandMask.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/ExpandMask.lean index e84e21e1b..5dff75b87 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/ExpandMask.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/ExpandMask.lean @@ -91,7 +91,8 @@ structure EAt (σ : State) (c g : Nat) (s : State) : Prop where theorem gpre {c g : Nat} (hg : g < 64) {s : State} (h : EAt σ c g s) : GPre c (X σ) ((spOf σ).at' 840) (σ.gpr .rdx) g s := by have hp' := spOk hp - refine ⟨h.rsi, h.rdi, hg, fun j hj => ?_, fun j hj => ?_, by rw [h.env.wr, hp.2.1]; simp, fun j hj hc => ?_⟩ + refine ⟨h.rsi, h.rdi, hg, fun j hj => ?_, fun j hj => ?_, + fun i hi => ⟨_, by rw [h.env.wr, hp.2.1]; simp, coeff_contains _ hi⟩, fun j hj hc => ?_⟩ · have := congrArg (fun L => L.getD j 0) h.out rw [MlKem.bytesAt_getD _ _ hj] at this exact this diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/ExpandMaskLoop.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/ExpandMaskLoop.lean index 1fbfda39c..636ee4d06 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/ExpandMaskLoop.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/ExpandMaskLoop.lean @@ -125,7 +125,7 @@ structure GPre (c : Nat) (X : List Byte) (out aP : Addr) (g : Nat) (s : State) : g_lt : g < 64 bytes : ∀ j < 640, s.mem (out + BitVec.ofNat 64 j) = X.getD j 0 rd : ∀ j ≤ 637, InRegions (s.rd ++ s.wr) (out + BitVec.ofNat 64 j) 4 - wr : pR aP ∈ s.wr + wr : ∀ i < 256, InRegions s.wr (coeffAddr aP i) 4 apart : ∀ j < 640, ¬ (pR aP).Contains (out + BitVec.ofNat 64 j) 1 /-- The 4 coefficients of group `g`. -/ @@ -151,7 +151,7 @@ theorem emGroup_ok {c : Nat} (hc : emOk c) {X : List Byte} {out aP : Addr} {g : have hkk : c * k / 8 ≤ c * 3 / 8 := Nat.div_le_div_right (Nat.mul_le_mul_left c (by omega)) have hg' : c / 2 * g ≤ c / 2 * 63 := Nat.mul_le_mul_left _ (by omega) refine WP.mono (emCoef_run c k (by omega) s' (by rw [hr, k'.2.1, k'.2.2]; exact h.rd _ (by omega)) - (by rw [ha, k'.2.2]; exact ⟨_, h.wr, coeff_contains _ (by omega)⟩)) fun s'' ⟨hm, k''⟩ => ?_ + (by rw [ha, k'.2.2]; exact h.wr _ (by omega))) fun s'' ⟨hm, k''⟩ => ?_ have hv := emCoef_val hc s'.mem (s'.gpr .rsi + BitVec.ofNat 64 (c * k / 8)) X (g := g) (k := k) (fun b hb => by rw [hr, offAdd, hf _ fun r hr' => by simp only [List.mem_singleton] at hr'; subst hr'; exact h.apart _ (by omega)] diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Absorb.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Absorb.lean new file mode 100644 index 000000000..0c8b09b73 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Absorb.lean @@ -0,0 +1,360 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.M4Base +import VerifiedGarbage.Proof.MlKem.X86_64.S4Absorb + +/-! +# ML-DSA on x86-64: `vg_mldsa_expand_mask_poly4_avx2`, the round constants and the padded seeds + +Untrusted: everything here is checked by Lean. After the prologue, the +table of the round constants (`rc_ok`), and the four states holding the +padded 66-byte seeds, byte by byte (`absorb_ok`), as for +`vg_mlkem_sample_ntt4_avx2` (`Proof/MlKem/X86_64/S4Absorb.lean`, whose +lemmas on the bytes of the states it uses). +-/ + +namespace VG.Proof.MlDsa.X86_64.Mask4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlDsa.X86_64.Sample.Mask4 +open VG.Impl.MlKem.X86_64.Sample4 (zero4 oRc) +open VG.Proof.MlKem.X86_64 +open VG.Proof.MlKem.X86_64.S4 (wb_in wb_out off4 bytes_write byte_setWidth) +open VG.Proof.Sha3.X86_64.X4 (q4 VUpd la ba Lanes4 wp_vmovq wp_vbcast wp_vst wp_vxor readW_write256 q4_ymm) +open VG.Proof.Sha3.X86_64 (wp_movi64) + +/-- `Env` after writes below the saved registers. -/ +theorem Env.write {σ s s' : State} (he : Env σ s) {a n : Nat} (h : a + n ≤ oSave) + (hf : Frame [⟨at' σ a, n⟩] s.mem s'.mem) (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) + (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .r14, .rsp, .r15], s'.gpr r = s.gpr r) : Env σ s' := by + refine ⟨hrd.trans he.rd, hwr.trans he.wr, by rw [hg .rbx (by simp), he.rbx], by rw [hg .r12 (by simp), he.r12], + by rw [hg .r13 (by simp), he.r13], by rw [hg .r14 (by simp), he.r14], by rw [hg .rsp (by simp), he.rsp], + by rw [hg .r15 (by simp), he.r15], + fun i hi => ?_, ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (by + simpa using Offset.disjoint (scr σ) (d := oSave + 8 * i) (n := 8) (e := a) (k := n) (by simp only [oSave] at h ⊢; omega) + (by simp only [oSave]; omega) (by simp only [oSave] at h; omega)) (by decide)] + exact he.saved i hi + · refine he.frame.trans (hf.sub fun r hr => ?_) + simp only [List.mem_singleton] at hr; subst hr + exact ⟨scrR σ, by simp, Offset.sub_base _ (by simp only [oSave] at h; omega)⟩ + +/-! ## The round constants -/ + +/-- After the first `n` round constants. -/ +structure RcInv (σ s₀ : State) (n : Nat) (s : State) : Prop where + keep : Keep [.rax] s₀ s + frame : Frame [⟨at' σ oRc, 768⟩] s₀.mem s.mem + rc : ∀ r < n, ∀ k < 4, s.mem.readW (la (scr σ) (50 + r) k) 64 = Spec.Sha3.RC r + +theorem rc_step {σ : State} (hp : Pre σ) {s₀ : State} (he : Env σ s₀) {r : Nat} (hr : r < 24) {s : State} + (h : RcInv σ s₀ r s) : + WP isa (.block [.movImm64 .rax (Spec.Sha3.RC r), .vop (.vmovq .xmm0 .rax), + .vop (.vpbroadcastq .l256 .xmm0 .xmm0), Impl.Sha3.X86_64.X4.st .rbx (oRc / 32 + r) .xmm0]) s + (RcInv σ s₀ (r + 1)) := by + have hbx : s.gpr .rbx = scr σ := by rw [h.keep.gpr (by decide), he.rbx] + refine wp_movi64 fun s₁ u₁ => wp_vmovq fun s₂ u₂ => wp_vbcast fun s₃ u₃ => + wp_vst (a := at' σ (32 * (50 + r))) (by rw [VG.Proof.Sha3.X86_64.ea_at, u₃.gpr, u₂.gpr, u₁.other _ (by decide), hbx]; rfl) + (by rw [u₃.wr, u₂.wr, u₁.wr, h.keep.2.2, he.wr]; exact in_scr hp rfl (by omega)) + fun s₄ g₄ _ m₄ r₄ w₄ => VG.Proof.Sha3.X86_64.wp_nil ?_ + have hv : ∀ k < 4, q4 s₃ .xmm0 k = Spec.Sha3.RC r := fun k hk => by + rw [u₃.val k hk, u₂.val 0 (by decide), ite_eq_left rfl, u₁.gpr] + have hm : s₄.mem = s.mem.writeW (at' σ (32 * (50 + r))) (s₃.ymm .xmm0) := by rw [m₄, u₃.mem, u₂.mem, u₁.mem] + refine ⟨⟨fun g hg => by rw [g₄, u₃.gpr, u₂.gpr, u₁.other g (by simpa using hg), h.keep.gpr hg], + by rw [r₄, u₃.rd, u₂.rd, u₁.rd, h.keep.2.1], by rw [w₄, u₃.wr, u₂.wr, u₁.wr, h.keep.2.2]⟩, ?_, + fun r' hr' k hk => ?_⟩ + · rw [hm] + exact h.frame.writeW (List.mem_singleton_self _) _ (Offset.contains _ (by simp only [oRc]; omega) + (by simp only [oRc]; omega) (by simp only [oRc]; omega)) + · rw [hm] + by_cases e : r' = r + · subst e + rw [la, ← Offset.add_add, readW_write256 _ _ _ hk, q4_ymm _ _ hk, hv k hk] + · have e := readW_writeW_off s.mem (scr σ) (s₃.ymm .xmm0) (d := 32 * (50 + r') + 8 * k) + (e := 32 * (50 + r)) (n := 8) (by omega) (by omega) (by omega) + exact e.trans (h.rc r' (by omega) k hk) + +theorem rcTable_eq : Impl.Sha3.X86_64.X4.rcTable .rbx (oRc / 32) = (List.range 24).flatMap fun r => + [.movImm64 .rax (Spec.Sha3.RC r), .vop (.vmovq .xmm0 .rax), .vop (.vpbroadcastq .l256 .xmm0 .xmm0), + Impl.Sha3.X86_64.X4.st .rbx (oRc / 32 + r) .xmm0] := rfl + +/-- The table of the round constants. -/ +theorem rc_ok {σ : State} (hp : Pre σ) {s₀ : State} (he : Env σ s₀) : + WP isa (.block (Impl.Sha3.X86_64.X4.rcTable .rbx (oRc / 32))) s₀ (RcInv σ s₀ 24) := by + rw [rcTable_eq] + exact wp_range_flatMap (M := isa) (RcInv σ s₀) (fun r s hr h => rc_step hp he hr h) 24 (Nat.le_refl _) s₀ + ⟨Keep.refl _ _, Frame.refl _ _, fun _ h => absurd h (by omega)⟩ + + +/-! ## The states, byte by byte -/ + +/-- The four states hold `F`. -/ +def SB (σ : State) (m : Mem) (F : Nat → Nat → Byte) : Prop := + ∀ k < 4, ∀ q < 200, m (ba (scr σ) k q) = F k q + +/-- During the writes to the states, after the round constants (in `m₁`). -/ +structure AI (σ : State) (m₁ : Mem) (s : State) : Prop where + env : Env σ s + frame : Frame [⟨scr σ, 800⟩] m₁ s.mem + +theorem AI.write {σ : State} {m₁ : Mem} {s s' : State} (h : AI σ m₁ s) {e n : Nat} (hn : e + n ≤ 800) + (hm : ∃ v : BitVec (8 * n), s'.mem = s.mem.writeW (at' σ e) v) (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) + (hg : ∀ r, r ≠ .rax → s'.gpr r = s.gpr r) : AI σ m₁ s' := by + obtain ⟨v, hv⟩ := hm + have hf : Frame [⟨at' σ e, n⟩] s.mem s'.mem := by + rw [hv]; exact (Frame.refl _ _).writeW (List.mem_singleton_self _) _ (by + rw [show 8 * n / 8 = n by omega]; exact Region.contains_self _ _) + refine ⟨h.env.write (by simp only [oSave]; omega) hf hrd hwr fun r hr => hg r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide), + h.frame.trans (hf.sub fun r hr => ?_)⟩ + simp only [List.mem_singleton] at hr; subst hr + exact ⟨_, List.mem_singleton_self _, Offset.sub_base _ hn⟩ + +/-! ### Zeroing -/ + +/-- After zeroing the first `n` lanes of each state. -/ +structure ZInv (σ : State) (m₁ : Mem) (n : Nat) (s : State) : Prop where + ai : AI σ m₁ s + x0 : ∀ k < 4, q4 s .xmm0 k = 0 + bytes : ∀ k < 4, ∀ q < 200, q / 8 < n → s.mem (ba (scr σ) k q) = 0 + +theorem zero_step {σ : State} (hp : Pre σ) {m₁ : Mem} {i : Nat} (hi : i < 25) {s : State} (h : ZInv σ m₁ i s) : + WP isa (.block [Impl.Sha3.X86_64.X4.st .rbx i .xmm0]) s (ZInv σ m₁ (i + 1)) := by + refine wp_vst (a := at' σ (32 * i)) (by rw [VG.Proof.Sha3.X86_64.ea_at, h.ai.env.rbx]) + (by rw [h.ai.env.wr]; exact in_scr hp rfl (by omega)) fun s' g' q' m' r' w' => VG.Proof.Sha3.X86_64.wp_nil ?_ + refine ⟨h.ai.write (e := 32 * i) (n := 32) (by omega) ⟨_, m'⟩ r' w' fun r _ => by rw [g'], + fun k hk => by rw [q', h.x0 k hk], fun k hk q hq hn => ?_⟩ + rw [m'] + have hb := bytes_write (m := s.mem) (p := scr σ) (F := fun k q => s.mem (ba (scr σ) k q)) + (fun _ _ _ _ => rfl) (s.ymm .xmm0) (e := 32 * i) (by decide) (by omega) hk hq + rw [hb] + split + · rename_i hc + simp only [off4] at hc ⊢ + rw [show 8 * (32 * (q / 8) + 8 * k + q % 8 - 32 * i) = 64 * k + 8 * (q % 8) by omega, ← extract_extract (s.ymm .xmm0) (64 * k) 64 (8 * (q % 8)) 8 (by omega), + q4_ymm _ _ hk, h.x0 k hk] + apply BitVec.eq_of_getLsbD_eq; intro j _; simp + · rename_i hc + simp only [off4] at hc + exact h.bytes k hk q hq (by omega) + +theorem zero4_eq : zero4 = Impl.Sha3.X86_64.X4.vb .vpxor .xmm0 .xmm0 .xmm0 :: + (List.range 25).flatMap fun i => [Impl.Sha3.X86_64.X4.st .rbx i .xmm0] := rfl + +theorem zero_ok {σ : State} (hp : Pre σ) {m₁ : Mem} {s : State} (h : AI σ m₁ s) : + WP isa (.block zero4) s (fun s' => AI σ m₁ s' ∧ SB σ s'.mem fun _ _ => 0) := by + rw [zero4_eq] + refine wp_vxor fun s₁ u₁ => WP.mono (wp_range_flatMap (M := isa) (ZInv σ m₁) (fun i s hi h => zero_step hp hi h) + 25 (Nat.le_refl _) s₁ ⟨h.write (e := 0) (n := 0) (by omega) ⟨0, ?_⟩ u₁.rd u₁.wr fun r _ => by rw [u₁.gpr], + fun k hk => by rw [u₁.val k hk, BitVec.xor_self]; rfl, fun _ _ _ _ h => absurd h (by omega)⟩) + fun s' h' => ⟨h'.ai, fun k hk q hq => h'.bytes k hk q hq (by omega)⟩ + rw [u₁.mem] + funext x + simp [Mem.writeW, Mem.write] + + +/-! ### The seeds -/ + +/-- Byte `q` of seed `k` (0 past its end). -/ +abbrev Bq (σ : State) (k q : Nat) : Byte := (B σ k).getD q 0 + +/-- The states after the first `K` seeds, and the first `i` lanes of seed `K`. -/ +def HF (σ : State) (K i : Nat) (k q : Nat) : Byte := + if (k < K ∧ q < 66) ∨ (k = K ∧ q < 8 * i) then Bq σ k q else 0 + +open VG.Proof.Sha3.X86_64 (wp_movm wp_store wp_movzx8 wp_store8 wp_mov32i wp_nil) in +theorem lane_step {σ : State} (hp : Pre σ) {m₁ : Mem} {K i : Nat} (hK : K < 4) (hi : i < 8) {s : State} + (h : AI σ m₁ s ∧ SB σ s.mem (HF σ K i)) : + WP isa (.block [.mov .rax (.mem (at_ .r12 (66 * K + 8 * i))), .store (at_ .rbx (32 * i + 8 * K)) .rax]) s + (fun s' => AI σ m₁ s' ∧ SB σ s'.mem (HF σ K (i + 1))) := by + obtain ⟨ha, hb⟩ := h + refine wp_movm (a := sd σ + BitVec.ofNat 64 (66 * K + 8 * i)) (by rw [ea_at, ha.env.r12]) + (in_sd' hp ha.env.rd ha.env.wr (by omega)) fun s₁ u₁ => wp_store (a := at' σ (32 * i + 8 * K)) + (by rw [ea_at, u₁.other _ (by decide), ha.env.rbx]) (by rw [u₁.wr]; exact in_scr hp ha.env.wr (by omega)) + fun s₂ g₂ m₂ r₂ w₂ => wp_nil ?_ + have hm : s₂.mem = s.mem.writeW (at' σ (32 * i + 8 * K)) (s.mem.readW (sd σ + BitVec.ofNat 64 (66 * K + 8 * i)) 64) := by + rw [m₂, u₁.mem, u₁.gpr] + refine ⟨ha.write (e := 32 * i + 8 * K) (n := 8) (by omega) ⟨_, hm⟩ (r₂.trans u₁.rd) (w₂.trans u₁.wr) + fun r hr => by rw [g₂, u₁.other r hr], fun k hk q hq => ?_⟩ + rw [hm, bytes_write hb _ (by decide) (by omega) hk hq] + simp only [off4] + by_cases hc : 32 * i + 8 * K ≤ 32 * (q / 8) + 8 * k + q % 8 ∧ 32 * (q / 8) + 8 * k + q % 8 < 32 * i + 8 * K + 64 / 8 + · have hk' : k = K := by omega + have hq' : q / 8 = i := by omega + subst hk' + rw [ifp hc, HF, ifp (.inr ⟨rfl, by omega⟩), + show 8 * (32 * (q / 8) + 8 * k + q % 8 - (32 * i + 8 * k)) = 8 * (q % 8) by omega, + byte_readW _ _ (by omega), Offset.add_add, show 66 * k + 8 * i + q % 8 = 66 * k + q by omega, + seed_byte hp ha.env.frame hK (by omega)] + · rw [ifn hc, HF, HF] + by_cases hc' : (k < K ∧ q < 66) ∨ (k = K ∧ q < 8 * i) + · rw [ifp hc', ifp (by omega)] + · rw [ifn hc', ifn (by omega)] + +open VG.Proof.Sha3.X86_64 (wp_movm wp_store wp_movzx8 wp_store8 wp_mov32i wp_nil) in +/-- Byte `64 + j` of seed `K`. -/ +theorem byte_step {σ : State} (hp : Pre σ) {m₁ : Mem} {K j : Nat} (hK : K < 4) (hj : j < 2) {s : State} + (h : AI σ m₁ s ∧ SB σ s.mem fun k q => if (k < K ∧ q < 66) ∨ (k = K ∧ q < 64 + j) then Bq σ k q else 0) : + WP isa (.block [.movzx8 .rax (at_ .r12 (66 * K + (64 + j))), .store8 (at_ .rbx (256 + 8 * K + j)) .rax]) s + (fun s' => AI σ m₁ s' ∧ + SB σ s'.mem fun k q => if (k < K ∧ q < 66) ∨ (k = K ∧ q < 64 + (j + 1)) then Bq σ k q else 0) := by + obtain ⟨ha, hb⟩ := h + refine wp_movzx8 (a := sd σ + BitVec.ofNat 64 (66 * K + (64 + j))) (by rw [ea_at, ha.env.r12]) + (in_sd' hp ha.env.rd ha.env.wr (by omega)) fun s₁ u₁ => wp_store8 (a := at' σ (256 + 8 * K + j)) + (by rw [ea_at, u₁.other _ (by decide), ha.env.rbx]) (by rw [u₁.wr]; exact in_scr hp ha.env.wr (by omega)) + fun s₂ g₂ m₂ r₂ w₂ => wp_nil ?_ + have hm : s₂.mem = s.mem.writeW (at' σ (256 + 8 * K + j)) (s.mem (sd σ + BitVec.ofNat 64 (66 * K + (64 + j)))) := by + rw [m₂, u₁.mem, u₁.gpr, byte_setWidth] + refine ⟨ha.write (e := 256 + 8 * K + j) (n := 1) (by omega) ⟨_, hm⟩ (r₂.trans u₁.rd) (w₂.trans u₁.wr) + fun r hr => by rw [g₂, u₁.other r hr], fun k hk q hq => ?_⟩ + rw [hm, bytes_write hb _ (by decide) (by omega) hk hq] + simp only [off4] + by_cases hc : 256 + 8 * K + j ≤ 32 * (q / 8) + 8 * k + q % 8 ∧ 32 * (q / 8) + 8 * k + q % 8 < 256 + 8 * K + j + 8 / 8 + · have hk' : k = K := by omega + have hq' : q = 64 + j := by omega + subst hk' hq' + rw [ifp hc, ifp (.inr ⟨rfl, by omega⟩), show 8 * (32 * ((64 + j) / 8) + 8 * k + (64 + j) % 8 - + (256 + 8 * k + j)) = 0 by omega, Proof.Sha3.extractLsb'_byte, seed_byte hp ha.env.frame hK (by omega)] + · rw [ifn hc] + by_cases hc' : (k < K ∧ q < 66) ∨ (k = K ∧ q < 64 + j) + · rw [ifp hc', ifp (by omega)] + · rw [ifn hc', ifn (by omega)] + +theorem seedLanes_eq (K : Nat) : seedLanes K = (List.range 8).flatMap (fun i => + [.mov .rax (.mem (at_ .r12 (66 * K + 8 * i))), .store (at_ .rbx (32 * i + 8 * K)) .rax]) ++ + (([.movzx8 .rax (at_ .r12 (66 * K + (64 + 0))), .store8 (at_ .rbx (256 + 8 * K + 0)) .rax] : List Instr) ++ + ([.movzx8 .rax (at_ .r12 (66 * K + (64 + 1))), .store8 (at_ .rbx (256 + 8 * K + 1)) .rax] : List Instr)) := by + simp only [seedLanes, Nat.add_zero]; rfl + +/-- The states after the first `K` seeds. -/ +def GF (σ : State) (K : Nat) (k q : Nat) : Byte := if k < K ∧ q < 66 then Bq σ k q else 0 + +theorem seed_step {σ : State} (hp : Pre σ) {m₁ : Mem} {K : Nat} (hK : K < 4) {s : State} + (h : AI σ m₁ s ∧ SB σ s.mem (GF σ K)) : + WP isa (.block (seedLanes K)) s (fun s' => AI σ m₁ s' ∧ SB σ s'.mem (GF σ (K + 1))) := by + rw [seedLanes_eq, WP.block_append_iff] + refine WP.mono (wp_range_flatMap (M := isa) (fun i s => AI σ m₁ s ∧ SB σ s.mem (HF σ K i)) + (fun i s hi h => lane_step hp hK hi h) 8 (Nat.le_refl _) s ⟨h.1, fun k hk q hq => ?_⟩) fun s₁ h₁ => ?_ + · rw [h.2 k hk q hq, GF, HF] + by_cases hc : k < K ∧ q < 66 + · rw [ifp hc, ifp (.inl hc)] + · rw [ifn hc, ifn (by omega)] + rw [WP.block_append_iff] + refine WP.mono (byte_step hp (j := 0) hK (by decide) ⟨h₁.1, fun k hk q hq => ?_⟩) fun s₂ h₂ => + WP.mono (byte_step hp (j := 1) hK (by decide) h₂) fun s₃ ⟨h₃, b₃⟩ => ⟨h₃, fun k hk q hq => ?_⟩ + · rw [h₁.2 k hk q hq, HF] + · rw [b₃ k hk q hq, GF] + dsimp only + by_cases hc : (k < K ∧ q < 66) ∨ (k = K ∧ q < 64 + (1 + 1)) + · rw [ifp hc] + by_cases hc' : k < K + 1 ∧ q < 66 + · rw [ifp hc'] + · rw [ifn hc']; omega + · rw [ifn hc] + by_cases hc' : k < K + 1 ∧ q < 66 + · omega + · rw [ifn hc'] + + +/-! ### The padding -/ + +open VG.Proof.Sha3.X86_64 (wp_store8 wp_mov32i wp_nil) in +/-- The byte `c` (in `rax`) to byte `q₀` of state `K`. -/ +theorem cbyte_step {σ : State} (hp : Pre σ) {m₁ : Mem} {F : Nat → Nat → Byte} {K q₀ : Nat} (hK : K < 4) + (hq₀ : q₀ < 200) {c : Byte} {s : State} (hax : s.gpr .rax = c.setWidth 64) (h : AI σ m₁ s ∧ SB σ s.mem F) : + WP isa (.block [.store8 (at_ .rbx (32 * (q₀ / 8) + 8 * K + q₀ % 8)) .rax]) s + (fun s' => AI σ m₁ s' ∧ s'.gpr .rax = s.gpr .rax ∧ + SB σ s'.mem fun k q => if k = K ∧ q = q₀ then c else F k q) := by + obtain ⟨ha, hb⟩ := h + refine wp_store8 (a := at' σ (32 * (q₀ / 8) + 8 * K + q₀ % 8)) (by rw [ea_at, ha.env.rbx]) + (by exact in_scr hp ha.env.wr (by omega)) fun s₂ g₂ m₂ r₂ w₂ => wp_nil ?_ + have hm : s₂.mem = s.mem.writeW (at' σ (32 * (q₀ / 8) + 8 * K + q₀ % 8)) c := by + rw [m₂, hax, byte_setWidth] + refine ⟨ha.write (n := 1) (by omega) ⟨_, hm⟩ r₂ w₂ fun r _ => by rw [g₂], by rw [g₂], fun k hk q hq => ?_⟩ + rw [hm, bytes_write hb _ (by decide) (by omega) hk hq] + simp only [off4] + by_cases hc : 32 * (q₀ / 8) + 8 * K + q₀ % 8 ≤ 32 * (q / 8) + 8 * k + q % 8 ∧ + 32 * (q / 8) + 8 * k + q % 8 < 32 * (q₀ / 8) + 8 * K + q₀ % 8 + 8 / 8 + · have e : k = K ∧ q = q₀ := by omega + rw [ifp hc, ifp e, show 8 * (32 * (q / 8) + 8 * k + q % 8 - (32 * (q₀ / 8) + 8 * K + q₀ % 8)) = 0 by omega, + Proof.Sha3.extractLsb'_byte] + · rw [ifn hc, ifn (by omega)] + +/-- The four states hold their padded seeds. -/ +def PF (σ : State) (k q : Nat) : Byte := + if q < 66 then Bq σ k q else if q = 66 then 0x1f else if q = 135 then 0x80 else 0 + +theorem sfx_eq : (List.range 4).flatMap (fun k => [Instr.store8 (at_ .rbx (256 + 8 * k + 2)) .rax]) = + (List.range 4).flatMap (fun k => [Instr.store8 (at_ .rbx (32 * (66 / 8) + 8 * k + 66 % 8)) .rax]) := rfl + +theorem last_eq : (List.range 4).flatMap (fun k => [Instr.store8 (at_ .rbx (512 + 8 * k + 7)) .rax]) = + (List.range 4).flatMap (fun k => [Instr.store8 (at_ .rbx (32 * (135 / 8) + 8 * k + 135 % 8)) .rax]) := rfl + +/-- The byte `c` to byte `q₀` of each state. -/ +theorem cbytes_ok {σ : State} (hp : Pre σ) {m₁ : Mem} {F : Nat → Nat → Byte} {q₀ : Nat} (hq₀ : q₀ < 200) + {c : Byte} {s : State} (hax : s.gpr .rax = c.setWidth 64) (h : AI σ m₁ s ∧ SB σ s.mem F) : + WP isa (.block ((List.range 4).flatMap fun k => [.store8 (at_ .rbx (32 * (q₀ / 8) + 8 * k + q₀ % 8)) .rax])) s + (fun s' => AI σ m₁ s' ∧ SB σ s'.mem fun k q => if q = q₀ then c else F k q) := by + refine WP.mono (wp_range_flatMap (M := isa) (fun K s' => AI σ m₁ s' ∧ s'.gpr .rax = c.setWidth 64 ∧ + SB σ s'.mem fun k q => if k < K ∧ q = q₀ then c else F k q) + (fun K s' hK ⟨ha, hx, hb⟩ => WP.mono (cbyte_step hp hK hq₀ hx ⟨ha, hb⟩) fun s'' ⟨ha', hx', hb'⟩ => + ⟨ha', hx'.trans hx, fun k hk q hq => ?_⟩) 4 (Nat.le_refl _) s ⟨h.1, hax, fun k hk q hq => ?_⟩) + fun s' ⟨ha, _, hb⟩ => ⟨ha, fun k hk q hq => ?_⟩ + · rw [hb' k hk q hq] + dsimp only + by_cases e : k = K ∧ q = q₀ + · rw [ifp e, ifp (by omega)] + · rw [ifn e] + by_cases e' : k < K ∧ q = q₀ + · rw [ifp e', ifp (by omega)] + · rw [ifn e', ifn (by omega)] + · rw [h.2 k hk q hq]; dsimp only; rw [ifn (by omega)] + · rw [hb k hk q hq] + dsimp only + by_cases e : q = q₀ + · rw [ifp e, ifp ⟨hk, e⟩] + · rw [ifn e, ifn (by omega)] + +theorem absorb4_eq : absorb4 = zero4 ++ ((List.range 4).flatMap seedLanes ++ + (([.mov32 .rax (.imm 0x1f)] : List Instr) ++ ((List.range 4).flatMap (fun k => + [Instr.store8 (at_ .rbx (32 * (66 / 8) + 8 * k + 66 % 8)) .rax]) ++ + (([.mov32 .rax (.imm 0x80)] : List Instr) ++ (List.range 4).flatMap (fun k => + [Instr.store8 (at_ .rbx (32 * (135 / 8) + 8 * k + 135 % 8)) .rax]))))) := by + simp only [absorb4, List.append_assoc, List.cons_append, List.nil_append] + +open VG.Proof.Sha3.X86_64 (wp_mov32i) in +/-- The padded seeds in the four states. -/ +theorem absorb_ok {σ : State} (hp : Pre σ) {m₁ : Mem} {s : State} (h : AI σ m₁ s) : + WP isa (.block absorb4) s (fun s' => AI σ m₁ s' ∧ SB σ s'.mem (PF σ)) := by + rw [absorb4_eq, WP.block_append_iff] + refine WP.mono (zero_ok hp h) fun s₁ ⟨h₁, z₁⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (wp_range_flatMap (M := isa) (fun K s => AI σ m₁ s ∧ SB σ s.mem (GF σ K)) + (fun K s hK h => seed_step hp hK h) 4 (Nat.le_refl _) s₁ ⟨h₁, fun k hk q hq => ?_⟩) fun s₂ h₂ => ?_ + · rw [z₁ k hk q hq, GF, ifn (by omega)] + rw [WP.block_append_iff] + refine WP.mono (wp_mov32i (is := []) fun s₃ u₃ => VG.Proof.Sha3.X86_64.wp_nil + (Q := fun s₃ => AI σ m₁ s₃ ∧ s₃.gpr .rax = (0x1f : Byte).setWidth 64 ∧ SB σ s₃.mem (GF σ 4)) + ⟨h₂.1.write (e := 0) (n := 0) (by omega) ⟨0, ?_⟩ u₃.rd u₃.wr fun r hr => u₃.other r hr, by rw [u₃.gpr]; rfl, + by rw [u₃.mem]; exact h₂.2⟩) fun s₃ ⟨a₃, x₃, b₃⟩ => ?_ + · rw [u₃.mem]; funext x; simp [Mem.writeW, Mem.write] + rw [WP.block_append_iff] + refine WP.mono (cbytes_ok hp (by decide) x₃ ⟨a₃, b₃⟩) fun s₄ ⟨a₄, b₄⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (wp_mov32i (is := []) fun s₅ u₅ => VG.Proof.Sha3.X86_64.wp_nil + (Q := fun s₅ => AI σ m₁ s₅ ∧ s₅.gpr .rax = (0x80 : Byte).setWidth 64 ∧ + SB σ s₅.mem fun k q => if q = 66 then 0x1f else GF σ 4 k q) + ⟨a₄.write (e := 0) (n := 0) (by omega) ⟨0, ?_⟩ u₅.rd u₅.wr fun r hr => u₅.other r hr, by rw [u₅.gpr]; rfl, + by rw [u₅.mem]; exact b₄⟩) fun s₅ ⟨a₅, x₅, b₅⟩ => ?_ + · rw [u₅.mem]; funext x; simp [Mem.writeW, Mem.write] + refine WP.mono (cbytes_ok hp (by decide) x₅ ⟨a₅, b₅⟩) fun s₆ ⟨a₆, b₆⟩ => ⟨a₆, fun k hk q hq => ?_⟩ + rw [b₆ k hk q hq, PF] + dsimp only + rw [GF] + by_cases e1 : q < 66 + · rw [ifn (show ¬ q = 135 by omega), ifn (show ¬ q = 66 by omega), ifp (show k < 4 ∧ q < 66 from ⟨hk, e1⟩), + ifp e1] + · rw [ifn e1, ifn (show ¬ (k < 4 ∧ q < 66) by omega)] + by_cases e2 : q = 66 + · rw [ifn (show ¬ q = 135 by omega), ifp e2, ifp e2] + · rw [ifn e2, ifn e2] + +end VG.Proof.MlDsa.X86_64.Mask4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Base.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Base.lean new file mode 100644 index 000000000..1075f88a5 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Base.lean @@ -0,0 +1,160 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.ExpandMask +import VerifiedGarbage.Proof.Sha3.X86_64.X4.Bytes +import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.ExpandMask4 + +/-! +# ML-DSA on x86-64: `vg_mldsa_expand_mask_poly4`, the layout + +Untrusted: everything here is checked by Lean. The contract the proofs are +written against (`em4K`), what holds between the pieces of the functions +(`Env`, relative to the entry state `σ`), and the prologue, as for +`vg_mlkem_sample_ntt4` (`Proof/MlKem/X86_64/S4Base.lean`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Mask4 + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Sample.Mask4 +open VG.Proof.MlKem.X86_64 (Keep WP.keep retR) +open VG.Proof.MlDsa.X86_64.Sample (gOf) +open VG.Spec.MlDsa (H PolyIs toRq bitUnpack bitlen seed66) +open VG.Spec.Sha3 (bytesAt) + +/-- The output polynomial `k` of four at `a`. -/ +abbrev poly4 (a : Addr) (k : Nat) : Addr := a + BitVec.ofNat 64 (1024 * k) + +/-- `vg_mldsa_expand_mask_poly4(seeds = rdi, gamma1 = esi, a = rdx, scratch = rcx)`, +with 24 bytes of stack below `rsp`. -/ +def em4K : Contract isa where + pre s := + s.rd = [⟨s.gpr .rdi, 264⟩] ∧ s.wr = [⟨s.gpr .rdx, 4096⟩, ⟨s.gpr .rcx, 8192⟩] ∧ + Region.Disjoint ⟨s.gpr .rdi, 264⟩ ⟨s.gpr .rdx, 4096⟩ ∧ Region.Disjoint ⟨s.gpr .rdi, 264⟩ ⟨s.gpr .rcx, 8192⟩ ∧ + Region.Disjoint ⟨s.gpr .rdx, 4096⟩ ⟨s.gpr .rcx, 8192⟩ ∧ + (retR s).Disjoint ⟨s.gpr .rdi, 264⟩ ∧ (retR s).Disjoint ⟨s.gpr .rdx, 4096⟩ ∧ + (retR s).Disjoint ⟨s.gpr .rcx, 8192⟩ ∧ + (below (s.gpr .rsp) 24).Disjoint ⟨s.gpr .rdi, 264⟩ ∧ (below (s.gpr .rsp) 24).Disjoint ⟨s.gpr .rdx, 4096⟩ ∧ + (below (s.gpr .rsp) 24).Disjoint ⟨s.gpr .rcx, 8192⟩ ∧ + (s.gpr .rdx).toNat + 4096 ≤ 2 ^ 64 ∧ (s.gpr .rcx).toNat + 8192 ≤ 2 ^ 64 ∧ + (gOf s = 2 ^ 17 ∨ gOf s = 2 ^ 19) + post s s' := ∀ k < 4, PolyIs s'.mem (poly4 (s.gpr .rdx) k) + (toRq (bitUnpack (H (seed66 s.mem (s.gpr .rdi) k) (32 * (1 + bitlen (gOf s - 1)))) (gOf s - 1) (gOf s))) + pub s₁ s₂ := s₁.gpr .rdi = s₂.gpr .rdi ∧ (s₁.gpr .rsi).setWidth 32 = (s₂.gpr .rsi).setWidth 32 ∧ + s₁.gpr .rdx = s₂.gpr .rdx ∧ s₁.gpr .rcx = s₂.gpr .rcx ∧ s₁.gpr .rsp = s₂.gpr .rsp + +section +variable (σ : State) +abbrev sd : Addr := σ.gpr .rdi +abbrev aP : Addr := σ.gpr .rdx +abbrev scr : Addr := σ.gpr .rcx +/-- Seed `k`. -/ +abbrev B (k : Nat) : List Byte := seed66 σ.mem (sd σ) k +abbrev sdR : Region := ⟨sd σ, 264⟩ +abbrev aR : Region := ⟨aP σ, 4096⟩ +abbrev scrR : Region := ⟨scr σ, 8192⟩ +abbrev stkR : Region := below (σ.gpr .rsp) 24 +/-- `scratch + off`. -/ +abbrev at' (off : Nat) : Addr := scr σ + BitVec.ofNat 64 off +end + +/-- The precondition, by name. -/ +structure Pre (σ : State) : Prop where + rd : σ.rd = [sdR σ] + wr : σ.wr = [aR σ, scrR σ] + sd_a : (sdR σ).Disjoint (aR σ) + sd_scr : (sdR σ).Disjoint (scrR σ) + a_scr : (aR σ).Disjoint (scrR σ) + ret_sd : (retR σ).Disjoint (sdR σ) + ret_a : (retR σ).Disjoint (aR σ) + ret_scr : (retR σ).Disjoint (scrR σ) + stk_sd : (stkR σ).Disjoint (sdR σ) + stk_a : (stkR σ).Disjoint (aR σ) + stk_scr : (stkR σ).Disjoint (scrR σ) + a_lt : (aP σ).toNat + 4096 ≤ 2 ^ 64 + scr_lt : (scr σ).toNat + 8192 ≤ 2 ^ 64 + gamma : gOf σ = 2 ^ 17 ∨ gOf σ = 2 ^ 19 + +theorem pre_of {σ : State} (h : em4K.pre σ) : Pre σ := + let ⟨h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11, h12, h13, h14⟩ := h + ⟨h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11, h12, h13, h14⟩ + +/-- What holds between the pieces. -/ +structure Env (σ s : State) : Prop where + rd : s.rd = σ.rd + wr : s.wr = σ.wr + rbx : s.gpr .rbx = scr σ + r12 : s.gpr .r12 = sd σ + r13 : s.gpr .r13 = aP σ + r14 : s.gpr .r14 = σ.gpr .rsi + rsp : s.gpr .rsp = σ.gpr .rsp + r15 : s.gpr .r15 = σ.gpr .r15 + saved : ∀ i < 5, s.mem.readW (at' σ (oSave + 8 * i)) 64 = σ.gpr (saved.getD i .rbx) + frame : Frame [aR σ, scrR σ, stkR σ] σ.mem s.mem + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +theorem sub_scr {a n : Nat} (h : a + n ≤ 8192) : Region.Sub ⟨at' σ a, n⟩ (scrR σ) := Offset.sub_base _ h + +omit hp in +theorem sub_poly {k : Nat} (hk : k < 4) : Region.Sub ⟨poly4 (aP σ) k, 1024⟩ (aR σ) := + Offset.sub_base _ (by omega) + +theorem in_scr {s : State} (hw : s.wr = σ.wr) {a n : Nat} (h : a + n ≤ 8192) : InRegions s.wr (at' σ a) n := by + rw [hw, hp.wr]; exact ⟨scrR σ, by simp, Offset.contains_base _ h (by omega)⟩ + +theorem in_scr' {s : State} (hr : s.rd = σ.rd) (hw : s.wr = σ.wr) {a n : Nat} (h : a + n ≤ 8192) : + InRegions (s.rd ++ s.wr) (at' σ a) n := by + rw [hr, hw, hp.rd, hp.wr]; exact ⟨scrR σ, by simp, Offset.contains_base _ h (by omega)⟩ + +theorem in_sd' {s : State} (hr : s.rd = σ.rd) (hw : s.wr = σ.wr) {a n : Nat} (h : a + n ≤ 264) : + InRegions (s.rd ++ s.wr) (sd σ + BitVec.ofNat 64 a) n := by + rw [hr, hw, hp.rd, hp.wr]; exact ⟨sdR σ, by simp, Offset.contains_base _ h (by omega)⟩ + +/-- The seeds are not written. -/ +theorem seeds_frame {m : Mem} (hf : Frame [aR σ, scrR σ, stkR σ] σ.mem m) {a : Nat} (ha : a < 264) : + m (sd σ + BitVec.ofNat 64 a) = σ.mem (sd σ + BitVec.ofNat 64 a) := + hf.bytes (R := sdR σ) (by simpa using ⟨hp.sd_a, hp.sd_scr, hp.stk_sd.symm⟩) (by simp) ha + +/-- Byte `a` of seed `k`. -/ +theorem seed_byte {m : Mem} (hf : Frame [aR σ, scrR σ, stkR σ] σ.mem m) {k a : Nat} (hk : k < 4) (ha : a < 66) : + m (sd σ + BitVec.ofNat 64 (66 * k + a)) = (B σ k).getD a 0 := by + rw [seeds_frame hp hf (by omega), B, seed66, ← Offset.add_add] + simp [bytesAt, ha] + +end + +/-! ## The prologue -/ + +/-- A read of 8 bytes at `scratch + d` after a write of 8 elsewhere. -/ +theorem rd64_off {σ : State} {m : Mem} {d e : Nat} {v : BitVec 64} (hd : d < 2 ^ 32) (he : e < 2 ^ 32) + (h : d + 8 ≤ e ∨ e + 8 ≤ d) : (m.writeW (at' σ e) v).readW (at' σ d) 64 = m.readW (at' σ d) 64 := + readW_writeW_off m _ v (n := 8) (by omega) (by omega) h + +theorem pro_eq : pro = [.store (VG.Impl.MlKem.X86_64.at_ .rcx 5088) .rbx, .store (VG.Impl.MlKem.X86_64.at_ .rcx 5096) .rbp, + .store (VG.Impl.MlKem.X86_64.at_ .rcx 5104) .r12, .store (VG.Impl.MlKem.X86_64.at_ .rcx 5112) .r13, + .store (VG.Impl.MlKem.X86_64.at_ .rcx 5120) .r14, .mov .rbx (.reg .rcx), .mov .r12 (.reg .rdi), + .mov .r13 (.reg .rdx), .mov .r14 (.reg .rsi)] := rfl + +theorem pro_ok {σ : State} (hp : Pre σ) : WP isa (.block pro) σ (Env σ) := by + rw [pro_eq] + refine WP.mono (WP.keep [.rbx, .r12, .r13, .r14] (Q := fun s => + s.mem = ((((σ.mem.writeW (at' σ 5088) (σ.gpr .rbx)).writeW (at' σ 5096) (σ.gpr .rbp)).writeW (at' σ 5104) + (σ.gpr .r12)).writeW (at' σ 5112) (σ.gpr .r13)).writeW (at' σ 5120) (σ.gpr .r14) ∧ + s.gpr .rbx = scr σ ∧ s.gpr .r12 = sd σ ∧ s.gpr .r13 = aP σ ∧ s.gpr .r14 = σ.gpr .rsi) + (by xrun [in_scr hp rfl (a := 5088) (n := 8) (by omega), in_scr hp rfl (a := 5096) (n := 8) (by omega), + in_scr hp rfl (a := 5104) (n := 8) (by omega), in_scr hp rfl (a := 5112) (n := 8) (by omega), + in_scr hp rfl (a := 5120) (n := 8) (by omega)]) + (by decide)) fun s1 ⟨⟨hm1, hbx, h12, h13, h14⟩, k1⟩ => ⟨k1.2.1, k1.2.2, hbx, h12, h13, h14, + k1.gpr (by decide), k1.gpr (by decide), fun i hi => ?_, ?_⟩ + · rw [hm1] + rcases (by omega : i = 0 ∨ i = 1 ∨ i = 2 ∨ i = 3 ∨ i = 4) with rfl | rfl | rfl | rfl | rfl <;> + simp (disch := omega) only [oSave, Mem.readW_writeW_self64, rd64_off, Nat.reduceMul, Nat.reduceAdd] <;> rfl + · rw [hm1] + have hin : ∀ a, a + 8 ≤ 8192 → (scrR σ).Contains (at' σ a) (64 / 8) := fun a ha => + Offset.contains_base _ (by omega) (by omega) + exact ((((((Frame.refl _ _).writeW (by simp) _ (hin 5088 (by omega))).writeW (by simp) _ + (hin 5096 (by omega))).writeW (by simp) _ (hin 5104 (by omega))).writeW (by simp) _ + (hin 5112 (by omega))).writeW (by simp) _ (hin 5120 (by omega))) + +end VG.Proof.MlDsa.X86_64.Mask4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4CT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4CT.lean new file mode 100644 index 000000000..597b797bc --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4CT.lean @@ -0,0 +1,128 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.M4Top + +/-! +# ML-DSA on x86-64: `vg_mldsa_expand_mask_poly4_avx2`, constant time + +Untrusted: everything here is checked by Lean. Two runs whose pointers and +`γ₁` agree (the declared public data) leak the same. The taint analysis +proves each piece from the pointers (the prologue, the absorption and the +squeezes, from the arguments and then `rbx`; the unpackings, from `rbx`, +`r13` and `rsp`); the two runs take the same branch on `γ₁` (`cmp_ok`), as +their `γ₁` agree. +-/ + +namespace VG.Proof.MlDsa.X86_64.Mask4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlDsa.X86_64.Sample.Mask4 +open VG.Impl.MlKem.X86_64.Sample4 (oRc) +open VG.Proof.MlKem.X86_64 +open VG.Proof.MlDsa.X86_64.Sample (emOk gOf) +open VG.Proof.MlDsa.Sample (emC) + +/-- Two runs related by `I`, from entry states that agree on what is public. -/ +abbrev R (I : State → State → Prop) : State → State → Prop := Rel2 em4K.pre em4K.pub I + +theorem env_rbx {σ₁ σ₂ s₁ s₂ : State} (hq : em4K.pub σ₁ σ₂) (e₁ : Env σ₁ s₁) (e₂ : Env σ₂ s₂) : + s₁.gpr .rbx = s₂.gpr .rbx := by rw [e₁.rbx, e₂.rbx, scr, scr, hq.2.2.2.1] + +theorem env_r13 {σ₁ σ₂ s₁ s₂ : State} (hq : em4K.pub σ₁ σ₂) (e₁ : Env σ₁ s₁) (e₂ : Env σ₂ s₂) : + s₁.gpr .r13 = s₂.gpr .r13 := by rw [e₁.r13, e₂.r13, aP, aP, hq.2.2.1] + +theorem env_rsp {σ₁ σ₂ s₁ s₂ : State} (hq : em4K.pub σ₁ σ₂) (e₁ : Env σ₁ s₁) (e₂ : Env σ₂ s₂) : + s₁.gpr .rsp = s₂.gpr .rsp := by rw [e₁.rsp, e₂.rsp, hq.2.2.2.2] + +/-- `squeeze4 n`, given its taint analysis. -/ +theorem sq_ct (n : Nat) (hn : n < 5) {hc : VG.Taint.Hint X86_64.Taint.T} + (c : (taint.check (X86_64.Taint.ofRegs [.rbx]) (squeeze4 n) hc).isSome = true) : + RelCT isa (R fun σ s => SqInv σ n s) (squeeze4 n) (R fun σ s => SqInv σ (n + 1) s) := + relInv (fun σ s hp h => sq_ok (pre_of hp) hn h) + (taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) c) + +/-- The branch on `γ₁`'s comparison, and what it keeps. -/ +theorem cmp_ok {σ : State} {s : State} (h : SqInv σ 5 s) : + WP isa (.block [.vop .vzeroupper, .alu32 .cmp .r14 (.imm 0x20000)]) s fun s' => + (∀ c, UI σ c 0 s') ∧ s'.zf = some (BitVec.setWidth 32 (σ.gpr .rsi) - 0x20000 == 0) := by + refine WP.mono (WP.keep [.r14] (Q := fun s' => s'.mem = s.mem ∧ s'.gpr .r14 = s.gpr .r14 ∧ + s'.zf = some (BitVec.setWidth 32 (s.gpr .r14) - 0x20000 == 0)) (by xrun; exact ⟨rfl, rfl, rfl⟩) (by decide)) + fun s1 ⟨⟨hm1, h14, hz1⟩, k1'⟩ => ⟨fun c => ⟨Env.low h.env (rs := []) (by simp) + (by rw [hm1]; exact Frame.refl _ _) k1'.2.1 k1'.2.2 fun r _ => by + by_cases e : r = .r14 + · subst e; exact h14 + · exact k1'.gpr (by simp [e]), + fun k hk p hp' => by rw [hm1]; exact h.buf k hk p (by omega), fun _ hk => absurd hk (by omega)⟩, + by rw [hz1, h.env.r14]⟩ + +/-- The comparison of `γ₁`, as `γ₁ = 2¹⁷`. -/ +theorem zf_gamma (σ : State) : + (BitVec.setWidth 32 (σ.gpr .rsi) - 0x20000 == 0) = decide (gOf σ = 2 ^ 17) := by + by_cases e : gOf σ = 2 ^ 17 + · rw [show BitVec.setWidth 32 (σ.gpr .rsi) = 0x20000 from BitVec.eq_of_toNat_eq (by rw [← gOf, e]; rfl), + decide_eq_true e] + rfl + · rw [decide_eq_false e, beq_eq_false_iff_ne] + intro h + apply e + rw [gOf, show BitVec.setWidth 32 (σ.gpr .rsi) = 0x20000 by + rw [← BitVec.sub_add_cancel (BitVec.setWidth 32 (σ.gpr .rsi)) 0x20000, h]; rfl] + rfl + +/-- A piece that takes each run from `I` to `I'`, keeping a fact `C` of the entry states. -/ +theorem relInvC {I I' : State → State → Prop} {C : State → Prop} {c : Prog isa} + (hw : ∀ σ s, em4K.pre σ → I σ s → WP isa c s (I' σ)) (ht : RelCT isa (R I) c fun _ _ => True) : + RelCT isa (R fun σ s => I σ s ∧ C σ) c (R fun σ s => I' σ s ∧ C σ) := + relInv (fun σ s hp hs => WP.mono (hw σ s hp hs.1) fun _ h => ⟨h, hs.2⟩) + (RelCT.mono ht (fun _ _ ⟨σ₁, σ₂, p₁, p₂, pub, i₁, i₂⟩ => ⟨σ₁, σ₂, p₁, p₂, pub, i₁.1, i₂.1⟩) fun _ _ h => h) + +/-- The unpackings for `c`, from states with the same pointers, keeping a fact `C` of the entry states. -/ +theorem unpack4_ct {c : Nat} (hc : emOk c) {C : State → Prop} {hh : VG.Taint.Hint X86_64.Taint.T} + (ht : (taint.check (X86_64.Taint.ofRegs [.rbx, .r13, .rsp]) (unpack4 c) hh).isSome = true) : + RelCT isa (R fun σ s => UI σ c 0 s ∧ C σ) (unpack4 c) (R fun σ s => UI σ c 4 s ∧ C σ) := + relInvC (fun σ s hp h => unpack4_ok (pre_of hp) hc h) + (taintRel [.rbx, .r13, .rsp] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl + exacts [env_rbx hq h₁.env h₂.env, env_r13 hq h₁.env h₂.env, env_rsp hq h₁.env h₂.env]) ht) + +theorem gOf_pub {σ₁ σ₂ : State} (hq : em4K.pub σ₁ σ₂) : gOf σ₁ = gOf σ₂ := by rw [gOf, gOf, hq.2.1] + +/-- The branch on `γ₁`. -/ +theorem sel_ct : RelCT isa (R fun σ s => (∀ c, UI σ c 0 s) ∧ + s.zf = some (BitVec.setWidth 32 (σ.gpr .rsi) - 0x20000 == 0)) + (.ite .e (unpack4 18) (unpack4 20)) (R fun σ s => UI σ (emC (gOf σ)) 4 s) := by + refine RelCT.ite (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ => by + show x.zf = y.zf; rw [h₁.2, h₂.2, hq.2.1]) ?_ ?_ + · refine RelCT.mono (unpack4_ct (C := fun σ => emC (gOf σ) = 18) (.inl rfl) (by taint_decide)) + (fun x y ⟨⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩, hb⟩ => ?_) + fun x y ⟨σ₁, σ₂, p₁, p₂, hq, ⟨u₁, c₁⟩, ⟨u₂, c₂⟩⟩ => ⟨σ₁, σ₂, p₁, p₂, hq, by show UI σ₁ _ 4 x; rw [c₁]; exact u₁, + by show UI σ₂ _ 4 y; rw [c₂]; exact u₂⟩ + have hb' : x.zf = some true := hb + rw [h₁.2, zf_gamma, Option.some.injEq, decide_eq_true_iff] at hb' + exact ⟨σ₁, σ₂, p₁, p₂, hq, ⟨h₁.1 18, by rw [hb']; rfl⟩, ⟨h₂.1 18, by rw [← gOf_pub hq, hb']; rfl⟩⟩ + · refine RelCT.mono (unpack4_ct (C := fun σ => emC (gOf σ) = 20) (.inr rfl) (by taint_decide)) + (fun x y ⟨⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩, hb⟩ => ?_) + fun x y ⟨σ₁, σ₂, p₁, p₂, hq, ⟨u₁, c₁⟩, ⟨u₂, c₂⟩⟩ => ⟨σ₁, σ₂, p₁, p₂, hq, by show UI σ₁ _ 4 x; rw [c₁]; exact u₁, + by show UI σ₂ _ 4 y; rw [c₂]; exact u₂⟩ + have hb' : x.zf = some false := hb + rw [h₁.2, zf_gamma, Option.some.injEq, decide_eq_false_iff_not] at hb' + have e₁ : gOf σ₁ = 2 ^ 19 := (pre_of p₁).gamma.resolve_left hb' + exact ⟨σ₁, σ₂, p₁, p₂, hq, ⟨h₁.1 20, by rw [e₁]; rfl⟩, ⟨h₂.1 20, by rw [← gOf_pub hq, e₁]; rfl⟩⟩ + +theorem ct : ConstantTime isa em4K.pre em4K.pub expandMask4Avx2 := by + unfold expandMask4Avx2 + refine relStart (Q := fun _ _ => True) (RelCT.seq (relInv (I' := fun σ s => SqInv σ 0 s) + (fun σ s hp h => by subst h; exact start_ok (pre_of hp)) + (taintRel [.rdi, .rdx, .rcx, .rsp] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + subst h₁ h₂ + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl + exacts [hq.1, hq.2.2.1, hq.2.2.2.1, hq.2.2.2.2]) (by taint_decide))) ?_) + refine RelCT.seq (sq_ct 0 (by decide) (by taint_decide)) (RelCT.seq (sq_ct 1 (by decide) (by taint_decide)) + (RelCT.seq (sq_ct 2 (by decide) (by taint_decide)) (RelCT.seq (sq_ct 3 (by decide) (by taint_decide)) + (RelCT.seq (sq_ct 4 (by decide) (by taint_decide)) ?_)))) + refine RelCT.seq (relInv (fun σ s _ h => cmp_ok h) (taintRel [] (fun _ _ _ _ hr => absurd hr List.not_mem_nil) + (by taint_decide))) (RelCT.seq sel_ct ?_) + exact taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) (by taint_decide) + +end VG.Proof.MlDsa.X86_64.Mask4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Scalar.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Scalar.lean new file mode 100644 index 000000000..750f5302a --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Scalar.lean @@ -0,0 +1,229 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.M4CT +import VerifiedGarbage.Proof.MlDsa.Arith.Mem +import VerifiedGarbage.Proof.MlKem.X86_64.ArithOk +import VerifiedGarbage.Proof.MlKem.X86_64.FragCall + +/-! +# ML-DSA on x86-64: `vg_mldsa_expand_mask_poly4` + +Untrusted: everything here is checked by Lean. The baseline implementation +calls `vg_mldsa_expand_mask_poly` on each seed, between the prologue and the +epilogue of the one for AVX2: after the call on seed `K`, polynomial `K` is +`ExpandMask`'s for the seed (`PC`), as in `vg_mldsa_rej_ntt_poly4` +(`Rej4Scalar.lean`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Mask4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlDsa.X86_64.Sample.Mask4 +open VG.Proof.MlKem.X86_64 +open VG.Proof.MlDsa.X86_64.Sample (emK gOf expandMask_correct expandMask_ct) +open VG.Proof.MlDsa.Arith (polyIs_frame) +open VG.Spec.MlDsa (PolyIs H toRq bitUnpack bitlen seed66) +open VG.Spec.Sha3 (bytesAt) + +theorem em_nosp : NoSp Impl.MlDsa.X86_64.Sample.expandMask := nosp_of (by decide +kernel) + +theorem em_depth : Impl.MlDsa.X86_64.Sample.expandMask.depth = 2 := by decide +kernel + +/-- `ExpandMask`'s polynomial for seed `k`. -/ +abbrev P (σ : State) (k : Nat) : Spec.MlDsa.Poly := + toRq (bitUnpack (H (B σ k) (32 * (1 + bitlen (gOf σ - 1)))) (gOf σ - 1) (gOf σ)) + +/-- Before the call on seed `K`. -/ +structure PC (σ : State) (K : Nat) (s : State) : Prop where + env : Env σ s + polys : ∀ k < K, PolyIs s.mem (poly4 (aP σ) k) (P σ k) + +/-- The regions of `vg_mldsa_expand_mask_poly`'s call for seed `K`. -/ +abbrev cRd (σ : State) (K : Nat) : List Region := [⟨sd σ + BitVec.ofNat 64 (66 * K), 66⟩] +abbrev cWr (σ : State) (K : Nat) : List Region := [pR (poly4 (aP σ) K), ⟨at' σ oScalar, 2048⟩] + +section +variable {σ : State} (hp : Pre σ) +include hp + +omit hp in +theorem c_sub {K : Nat} (hK : K < 4) : ∀ r ∈ cWr σ K ++ [stkR σ], + (∃ R ∈ [aR σ, scrR σ, stkR σ], Region.Sub r R) := by + intro r hr + simp only [List.mem_append, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with (rfl | rfl) | rfl + · exact ⟨aR σ, by simp, sub_poly hK⟩ + · exact ⟨scrR σ, by simp, sub_scr (by simp only [oScalar]; omega)⟩ + · exact ⟨stkR σ, by simp, fun _ h => h⟩ + +/-- A region the call writes is apart from `⟨at' σ a, n⟩` in the scratch space below 6144. -/ +theorem c_disj {K : Nat} (hK : K < 4) {a n : Nat} (h : a + n ≤ oScalar) : + ∀ r ∈ cWr σ K ++ [stkR σ], Region.Disjoint ⟨at' σ a, n⟩ r := by + intro r hr + simp only [List.mem_append, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with (rfl | rfl) | rfl + · exact (hp.a_scr.symm.sub_left (sub_scr (by simp only [oScalar] at h; omega))).sub_right (sub_poly hK) + · exact Offset.disjoint _ (.inl h) (by simp only [oScalar] at h; omega) (by simp only [oScalar]; omega) + · exact (hp.stk_scr.sub_right (sub_scr (by simp only [oScalar] at h; omega))).symm + +theorem seed_bytes {K : Nat} (hK : K < 4) {m : Mem} (hf : Frame [aR σ, scrR σ, stkR σ] σ.mem m) : + bytesAt m (sd σ + BitVec.ofNat 64 (66 * K)) 66 = B σ K := by + rw [MlKem.bytesAt_frame hf (by simpa using ⟨hp.sd_a.sub_left (Offset.sub_base _ (by omega)), + hp.sd_scr.sub_left (Offset.sub_base _ (by omega)), (hp.stk_sd.sub_right (Offset.sub_base _ (by omega))).symm⟩) + (by decide)] + rfl + +theorem scr6144_lt : (at' σ oScalar).toNat + 2048 ≤ 2 ^ 64 := by + have := hp.scr_lt + rw [at', Offset.toNat_add_ofNat, Nat.mod_eq_of_lt (show oScalar < 2 ^ 64 by decide), + Nat.mod_eq_of_lt (by simp only [oScalar]; omega)] + simp only [oScalar]; omega + +theorem regs {s : State} (he : Env σ s) : s.rd ++ s.wr = [sdR σ, aR σ, scrR σ] := by + rw [he.rd, he.wr, hp.rd, hp.wr]; rfl + +theorem cov {s : State} (he : Env σ s) {K : Nat} (hK : K < 4) : + Covers (cRd σ K ++ cWr σ K) (s.rd ++ s.wr) ∧ Covers (cWr σ K) s.wr := by + refine ⟨Covers.of_sub fun r hr => ?_, Covers.of_sub fun r hr => ?_⟩ + · rw [regs hp he] + simp only [List.cons_append, List.nil_append, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl + · exact ⟨sdR σ, by simp, 66 * K, rfl, by simp only; omega⟩ + · exact ⟨aR σ, by simp, 1024 * K, rfl, by simp only; omega⟩ + · exact ⟨scrR σ, by simp, oScalar, rfl, by simp only [oScalar]; omega⟩ + · rw [he.wr, hp.wr] + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl + · exact ⟨aR σ, by simp, 1024 * K, rfl, by simp only; omega⟩ + · exact ⟨scrR σ, by simp, oScalar, rfl, by simp only [oScalar]; omega⟩ + +/-- `PC` after the call. -/ +theorem PC.call {K : Nat} (hK : K < 4) {s s' : State} (h : PC σ K s) (hrd : s'.rd = s.rd) + (hwr : s'.wr = s.wr) (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .r14, .rsp, .r15], s'.gpr r = s.gpr r) + (hf : Frame (cWr σ K ++ [stkR σ]) s.mem s'.mem) : PC σ K s' := by + refine ⟨⟨hrd.trans h.env.rd, hwr.trans h.env.wr, by rw [hg .rbx (by simp), h.env.rbx], + by rw [hg .r12 (by simp), h.env.r12], by rw [hg .r13 (by simp), h.env.r13], by rw [hg .r14 (by simp), h.env.r14], + by rw [hg .rsp (by simp), h.env.rsp], by rw [hg .r15 (by simp), h.env.r15], fun i hi => ?_, + h.env.frame.trans (hf.sub (c_sub (σ := σ) hK))⟩, fun k hk => ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (c_disj hp hK (by simp only [oSave, oScalar]; omega)) (by decide)] + exact h.env.saved i hi + · refine polyIs_frame hf (fun r hr => ?_) (h.polys k hk) + simp only [List.mem_append, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with (rfl | rfl) | rfl + · have hd := Offset.disjoint (aP σ) (d := 1024 * k) (n := 1024) (e := 1024 * K) (k := 1024) (by omega) (by omega) + (by omega) + simpa [poly4] using hd + · exact (hp.a_scr.sub_left (sub_poly (by omega))).sub_right (sub_scr (by simp only [oScalar]; omega)) + · exact (hp.stk_a.sub_right (sub_poly (by omega))).symm + +/-- The arguments of the call on seed `K`. -/ +structure ArgI (σ : State) (K : Nat) (s : State) : Prop where + pinv : PC σ K s + rdi : s.gpr .rdi = sd σ + BitVec.ofNat 64 (66 * K) + rsi : s.gpr .rsi = σ.gpr .rsi + rdx : s.gpr .rdx = poly4 (aP σ) K + rcx : s.gpr .rcx = at' σ oScalar + +omit hp in +theorem sx6144 : BitVec.signExtend 64 (BitVec.ofNat 32 oScalar) = BitVec.ofNat 64 6144 := by decide + +omit hp in +theorem argsK_ok {K : Nat} (hK : K < 4) {s : State} (h : PC σ K s) : + WP isa (.block [.mov .rdi (.reg .r12), .alu .add .rdi (.imm (BitVec.ofNat 32 (66 * K))), .mov .rsi (.reg .r14), + .mov .rdx (.reg .r13), .alu .add .rdx (.imm (BitVec.ofNat 32 (1024 * K))), .mov .rcx (.reg .rbx), + .alu .add .rcx (.imm (BitVec.ofNat 32 oScalar))]) s (ArgI σ K) := + WP.mono (WP.keep [.rdi, .rsi, .rdx, .rcx] (Q := fun s' => s'.mem = s.mem ∧ + s'.gpr .rdi = sd σ + BitVec.ofNat 64 (66 * K) ∧ s'.gpr .rsi = σ.gpr .rsi ∧ s'.gpr .rdx = poly4 (aP σ) K ∧ + s'.gpr .rcx = at' σ oScalar) + (by xrun [h.env.r12, h.env.r13, h.env.r14, h.env.rbx, sx_ofNat (show 66 * K < 2 ^ 31 by omega), + sx_ofNat (show 1024 * K < 2 ^ 31 by omega), sx6144]; rfl) (by rfl)) + fun _ ⟨⟨hm₂, hdi, hsi, hdx, hcx⟩, k₂⟩ => ⟨⟨h.env.keep hm₂ k₂ (by decide), by rw [hm₂]; exact h.polys⟩, + hdi, hsi, hdx, hcx⟩ + +theorem argK_kS {K : Nat} (hK : K < 4) {s : State} (h : ArgI σ K s) : + (below (s.gpr .rsp) 24).Disjoint ⟨sd σ + BitVec.ofNat 64 (66 * K), 66⟩ := by + rw [h.pinv.env.rsp]; exact hp.stk_sd.sub_right (Offset.sub_base (sd σ) (d := 66 * K) (n := 66) (by omega)) + +theorem argK_pre {K : Nat} (hK : K < 4) {s : State} (h : ArgI σ K s) : + emK.pre (s.callEntry.withRegions (cRd σ K) (cWr σ K)) := by + have hsp : s.gpr .rsp = σ.gpr .rsp := h.pinv.env.rsp + have kS := argK_kS hp hK h + have kA : (below (s.gpr .rsp) 24).Disjoint (pR (poly4 (aP σ) K)) := by + rw [hsp]; exact hp.stk_a.sub_right (sub_poly (σ := σ) hK) + have kZ : (below (s.gpr .rsp) 24).Disjoint ⟨at' σ oScalar, 2048⟩ := by + rw [hsp]; exact hp.stk_scr.sub_right (sub_scr (σ := σ) (a := oScalar) (n := 2048) (by simp only [oScalar]; omega)) + simp only [emK, gOf, State.withRegions_gpr, State.withRegions_rd, State.withRegions_wr, + ce_gpr' s (by decide : Reg.rdi ≠ .rsp), ce_gpr' s (by decide : Reg.rsi ≠ .rsp), + ce_gpr' s (by decide : Reg.rdx ≠ .rsp), ce_gpr' s (by decide : Reg.rcx ≠ .rsp), h.rdi, h.rsi, h.rdx, h.rcx] + exact ⟨trivial, trivial, + (hp.sd_a.sub_left (Offset.sub_base _ (by omega))).sub_right (sub_poly hK), + (hp.sd_scr.sub_left (Offset.sub_base _ (by omega))).sub_right (sub_scr (by simp only [oScalar]; omega)), + (hp.a_scr.sub_left (sub_poly hK)).sub_right (sub_scr (by simp only [oScalar]; omega)), + ret_disj24 s kS, ret_disj24 s kA, ret_disj24 s kZ, stk_disj24' s kS, stk_disj24' s kA, stk_disj24' s kZ, + scr6144_lt hp, hp.gamma⟩ + +theorem callK_ok {K : Nat} (hK : K < 4) {s : State} (h : ArgI σ K s) : + WP isa (.call "vg_mldsa_expand_mask_poly" Impl.MlDsa.X86_64.Sample.expandMask) s (PC σ (K + 1)) := by + have hcv := cov hp h.pinv.env hK + refine WP.call expandMask_correct em_nosp (by rw [em_depth]; decide) (argK_pre hp hK h) hcv.1 hcv.2 + fun s₃ hrd hwr hcs hf _ ⟨s₃', hm₃, _, hpost⟩ => ?_ + rw [em_depth, h.pinv.env.rsp] at hf + have h₃ : PC σ K s₃ := h.pinv.call hp hK hrd hwr (fun r hr => hcs r (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide)) hf + simp only [emK, gOf, State.withRegions_gpr, State.withRegions_mem, ce_gpr' s (by decide : Reg.rdi ≠ .rsp), + ce_gpr' s (by decide : Reg.rsi ≠ .rsp), ce_gpr' s (by decide : Reg.rdx ≠ .rsp), h.rdi, h.rsi, h.rdx, hm₃, + ce_bytesAt24 s (n := 66) (by decide) (argK_kS hp hK h), seed_bytes hp hK h.pinv.env.frame] at hpost + refine ⟨h₃.env, fun k hk => ?_⟩ + by_cases e : k = K + · subst e; exact hpost + · exact h₃.polys k (by omega) + +theorem callK_ok' {K : Nat} (hK : K < 4) {s : State} (h : PC σ K s) : WP isa (callK K) s (PC σ (K + 1)) := + WP.seq (WP.mono (argsK_ok hK h) fun _ h₂ => callK_ok hp hK h₂) + +omit hp in +theorem epi_eq' : epi = [.mov .r14 (.mem (at_ .rbx 5120)), .mov .r13 (.mem (at_ .rbx 5112)), + .mov .r12 (.mem (at_ .rbx 5104)), .mov .rbp (.mem (at_ .rbx 5096)), .mov .rbx (.mem (at_ .rbx 5088))] := rfl + +/-- The postcondition, and the callee-saved registers restored. -/ +theorem endS_ok {s : State} (h : PC σ 4 s) : + WP isa (.block epi) s fun s' => em4K.post σ s' ∧ gprPreserved σ s' := by + have hin : ∀ i < 5, InRegions (s.rd ++ s.wr) (scr σ + BitVec.ofNat 64 (oSave + 8 * i)) 8 := fun i hi => + in_scr' hp h.env.rd h.env.wr (by simp only [oSave]; omega) + rw [epi_eq'] + refine WP.mono (WP.keep [.r14, .r13, .r12, .rbp, .rbx] (Q := fun s' => s'.mem = s.mem ∧ + s'.gpr .r14 = s.mem.readW (at' σ (oSave + 8 * 4)) 64 ∧ s'.gpr .r13 = s.mem.readW (at' σ (oSave + 8 * 3)) 64 ∧ + s'.gpr .r12 = s.mem.readW (at' σ (oSave + 8 * 2)) 64 ∧ s'.gpr .rbp = s.mem.readW (at' σ (oSave + 8 * 1)) 64 ∧ + s'.gpr .rbx = s.mem.readW (at' σ (oSave + 8 * 0)) 64) + (by + have h0 := hin 0 (by decide); have h1 := hin 1 (by decide); have h2 := hin 2 (by decide) + have h3 := hin 3 (by decide); have h4 := hin 4 (by decide) + simp only [oSave, Nat.reduceMul, Nat.reduceAdd] at h0 h1 h2 h3 h4 + xrun [h0, h1, h2, h3, h4, h.env.rbx] + exact ⟨rfl, rfl, rfl, rfl, rfl⟩) + (by decide)) fun s' ⟨⟨hm, h14, h13, h12, hbp, hbx⟩, k⟩ => ?_ + refine ⟨fun K hK => by rw [hm]; exact h.polys K hK, fun r hr => ?_, ?_⟩ + · simp only [calleeSaved, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl | rfl + · rw [hbx]; exact h.env.saved 0 (by decide) + · rw [hbp]; exact h.env.saved 1 (by decide) + · rw [k.gpr (by decide)]; exact h.env.rsp + · rw [h12]; exact h.env.saved 2 (by decide) + · rw [h13]; exact h.env.saved 3 (by decide) + · rw [h14]; exact h.env.saved 4 (by decide) + · rw [k.gpr (by decide)]; exact h.env.r15 + · rw [hm] + exact h.env.frame.readW (Region.contains_self _ _) (by + simpa using ⟨hp.ret_a, hp.ret_scr, Offset.base_disjoint_below (σ.gpr .rsp) (n := 24) (k := 8) (by omega)⟩) + (by decide) + +end + +theorem correct_scalar (σ : State) (hs : em4K.pre σ) : + ∃ t s', Exec isa expandMask4 σ t s' ∧ abiPreserved σ s' ∧ em4K.post σ s' := by + have hp := pre_of hs + obtain ⟨t, s', he, hF⟩ := WP.seq (WP.mono (pro_ok hp) fun _ h => + WP.seq (WP.mono (callK_ok' hp (by decide) (⟨h, fun _ h' => absurd h' (by omega)⟩ : PC σ 0 _)) fun _ p₁ => + WP.seq (WP.mono (callK_ok' hp (by decide) p₁) fun _ p₂ => WP.seq (WP.mono (callK_ok' hp (by decide) p₂) + fun _ p₃ => WP.seq (WP.mono (callK_ok' hp (by decide) p₃) fun _ p₄ => endS_ok hp p₄))))) + exact ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he hF.2, hF.1⟩ + +end VG.Proof.MlDsa.X86_64.Mask4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Squeeze.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Squeeze.lean new file mode 100644 index 000000000..68b2c42b7 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Squeeze.lean @@ -0,0 +1,253 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.M4Absorb +import VerifiedGarbage.Proof.MlKem.X86_64.S4Squeeze + +/-! +# ML-DSA on x86-64: `vg_mldsa_expand_mask_poly4_avx2`, squeezing + +Untrusted: everything here is checked by Lean. The padded seeds are the +states that `Keccak-f` turns into the absorbed ones (`padded_A0`), and the +four states hold them after `absorb4` (`lanes_A0`). Each `squeeze4 n` +permutes the four states (`permute4_ok`) and copies the first 136 bytes of +each to its output, which then holds the first `136 (n + 1)` bytes of +SHAKE256 of the seed (`hByte`, `sq_ok`), as for `vg_mlkem_sample_ntt4_avx2` +(`Proof/MlKem/X86_64/S4Squeeze.lean`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Mask4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlDsa.X86_64.Sample.Mask4 +open VG.Impl.MlKem.X86_64.Sample4 (permArgs oBuf) +open VG.Proof.MlKem.X86_64 +open VG.Proof.MlKem.X86_64.S4 (wb_in wb_out sx800 sx1600 sx2368) +open VG.Spec.Sha3 (keccakF RC) +open VG.Proof.Sha3 (byteOf Rep xorByte byteOf_xorByte byteOf_xorBytes absorb_pad iterF iterF_succ iterF_keccakF) +open VG.Proof.MlKem (padded shake256_eq) +open VG.Proof.Sha3 (squeeze_getElem length_squeeze) +open VG.Proof.Sha3.X86_64.X4 (la ba Lanes4 lanes4_of_bytes byte_of_lanes4 Pre4 permute4_ok) + +/-! ## The padded seeds -/ + +/-- The state whose permutation is the absorbed padded seed. -/ +def A0 (Bs : List Byte) : Spec.Sha3.State := xorByte (xorByte (Rep 136 Bs) 66 0x1f) 135 0x80 + +theorem padded_A0 {Bs : List Byte} (h : Bs.length = 66) : + padded 136 Spec.Sha3.shakeSuffix Bs = keccakF (A0 Bs) := by + rw [padded, absorb_pad (by decide) (by decide), h]; rfl + +theorem byteOf_A0 {Bs : List Byte} (h : Bs.length = 66) {q : Nat} (hq : q < 200) : + byteOf (A0 Bs) q = if q < 66 then Bs.getD q 0 else if q = 66 then 0x1f else if q = 135 then 0x80 else 0 := by + have hz : byteOf Spec.Sha3.zero q = 0 := by + simp only [byteOf, Spec.Sha3.zero, getElem!_pos (Vector.replicate 25 (0 : BitVec 64)) (q / 8) (by omega), + Vector.getElem_replicate] + apply BitVec.eq_of_getLsbD_eq; intro j _; simp + have ha : Spec.Sha3.absorb 136 Bs = Spec.Sha3.zero := by simp [Spec.Sha3.absorb, h] + have hr : byteOf (Rep 136 Bs) q = Bs.getD q 0 := by + rw [Rep, ha, byteOf_xorBytes _ _ hq, hz, h, show 136 * (66 / 136) = 0 from rfl, List.drop_zero] + exact BitVec.zero_xor + have hd : ∀ q, 66 ≤ q → Bs.getD q 0 = 0 := fun q hq' => by + rw [List.getD, List.getElem?_eq_none (by omega)]; rfl + rw [A0, byteOf_xorByte _ _ _ hq, byteOf_xorByte _ _ _ hq, hr] + by_cases e1 : q < 66 + · rw [ifn (show ¬ q = 135 by omega), ifn (show ¬ q = 66 by omega), ifp e1] + · rw [ifn e1, hd q (by omega)] + by_cases e2 : q = 66 + · rw [ifn (show ¬ q = 135 by omega), ifp e2, ifp e2]; exact BitVec.zero_xor + · rw [ifn e2, ifn e2] + by_cases e3 : q = 135 + · rw [ifp e3, ifp e3]; exact BitVec.zero_xor + · rw [ifn e3, ifn e3] + +theorem B_length (σ : State) (k : Nat) : (B σ k).length = 66 := VG.Proof.Sha3.bytesAt_length _ _ _ + +/-- Byte `p` of SHAKE256 of `B`: byte `p mod 136` of the padded state after +`⌊p / 136⌋` more permutations. -/ +def hByte (B : List Byte) (p : Nat) : Byte := + byteOf (iterF (p / 136) (padded 136 Spec.Sha3.shakeSuffix B)) (p % 136) + +theorem H_getD (B : List Byte) {ℓ p : Nat} (hp : p < ℓ) : (Spec.MlDsa.H B ℓ).getD p 0 = hByte B p := by + have hl : (Spec.MlDsa.H B ℓ).length = ℓ := length_squeeze (by decide) (by decide) _ _ + rw [List.getD_eq_getElem?_getD, List.getElem?_eq_getElem (by rw [hl]; exact hp), Option.getD_some] + exact squeeze_getElem (by decide) (by decide) _ hp + +/-- Byte `p` of SHAKE256 of `Bs`, from the states. -/ +theorem hByte_A0 {Bs : List Byte} (h : Bs.length = 66) {n p : Nat} (hp : p < 136) : + hByte Bs (136 * n + p) = byteOf (iterF (n + 1) (A0 Bs)) p := by + rw [hByte, show (136 * n + p) / 136 = n by omega, show (136 * n + p) % 136 = p by omega, padded_A0 h, + iterF_keccakF] + +/-! ## `Env` after writes -/ + +/-- `Env` after writes below the saved registers. -/ +theorem Env.low {σ s s' : State} (he : Env σ s) {rs : List Region} (hrs : ∀ r ∈ rs, Region.Sub r ⟨scr σ, oSave⟩) + (hf : Frame rs s.mem s'.mem) (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) + (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .r14, .rsp, .r15], s'.gpr r = s.gpr r) : Env σ s' := by + refine ⟨hrd.trans he.rd, hwr.trans he.wr, by rw [hg .rbx (by simp), he.rbx], by rw [hg .r12 (by simp), he.r12], + by rw [hg .r13 (by simp), he.r13], by rw [hg .r14 (by simp), he.r14], by rw [hg .rsp (by simp), he.rsp], + by rw [hg .r15 (by simp), he.r15], + fun i hi => ?_, ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (fun r hr => ((Offset.base_disjoint (scr σ) (k := oSave) + (e := oSave + 8 * i) (n := 8) (by omega) (by simp only [oSave]; omega)).symm).sub_right (hrs r hr)) + (by decide)] + exact he.saved i hi + · refine he.frame.trans (hf.sub fun r hr => ⟨scrR σ, by simp, fun x hx => Offset.sub_base (scr σ) (d := 0) + (n := oSave) (k := 8192) (by simp only [oSave]; omega) x (by rw [BitVec.add_zero]; exact hrs r hr x hx)⟩) + +/-- `Env` after code that writes no memory and keeps its registers. -/ +theorem Env.keep {σ s s' : State} (he : Env σ s) (hm : s'.mem = s.mem) {rs : List Reg} (hk : Keep rs s s') + (hrs : ∀ r ∈ [Reg.rbx, .r12, .r13, .r14, .rsp, .r15], r ∉ rs) : Env σ s' := + Env.low he (rs := []) (by simp) (by rw [hm]; exact Frame.refl _ _) hk.2.1 hk.2.2 fun r hr => hk.gpr (hrs r hr) + +/-! ## The squeezes -/ + +/-- After `n` squeezes. -/ +structure SqInv (σ : State) (n : Nat) (s : State) : Prop where + env : Env σ s + rc : ∀ r < 24, ∀ k < 4, s.mem.readW (la (scr σ) (50 + r) k) 64 = RC r + lanes : Lanes4 s.mem (scr σ) fun k => iterF n (A0 (B σ k)) + buf : ∀ k < 4, ∀ p < 136 * n, s.mem (at' σ (oBuf + 680 * k + p)) = hByte (B σ k) p + +theorem lanes_A0 {σ : State} {m : Mem} (h : SB σ m (PF σ)) : Lanes4 m (scr σ) fun k => iterF 0 (A0 (B σ k)) := + lanes4_of_bytes fun k hk q hq => by + rw [h k hk q hq, show iterF 0 (A0 (B σ k)) = A0 (B σ k) from rfl, byteOf_A0 (B_length σ k) hq, PF] + + + +theorem la_tbl (σ : State) (r k : Nat) : la (at' σ 1600) r k = la (scr σ) (50 + r) k := by + rw [la, la, at', Offset.add_add, show 1600 + (32 * r + 8 * k) = 32 * (50 + r) + 8 * k by omega] + +theorem args_ok {σ s : State} (he : Env σ s) : + WP isa (.block permArgs) s fun s' => (s'.mem = s.mem ∧ s'.gpr .rdi = scr σ ∧ s'.gpr .rsi = at' σ 800 ∧ + s'.gpr .rdx = at' σ 1600 ∧ s'.gpr .rcx = at' σ 2368) ∧ Keep [.rdi, .rsi, .rdx, .rcx] s s' := by + refine WP.keep _ ?_ (by decide) + unfold permArgs + xrun [he.rbx, sx800, sx1600, sx2368] + +/-- The permutation's precondition, in the scratch space. -/ +theorem pre4 {σ : State} (hp : Pre σ) {s : State} (hrd : s.rd = σ.rd) (hwr : s.wr = σ.wr) + (hrc : ∀ r < 24, ∀ k < 4, s.mem.readW (la (scr σ) (50 + r) k) 64 = RC r) : + Pre4 s (scr σ) (at' σ 800) (at' σ 1600) := + ⟨fun i _ => in_scr hp hwr (a := 32 * i) (by omega), + fun i _ => by rw [at', Offset.add_add]; exact in_scr hp hwr (by omega), + fun r _ => by rw [at', Offset.add_add]; exact in_scr' hp hrd hwr (by omega), + Offset.base_disjoint _ (by omega) (by omega), + Offset.base_disjoint _ (by omega) (by omega), + Offset.disjoint _ (by omega) (by omega) (by omega), + fun r hr k hk => by rw [la_tbl]; exact hrc r hr k hk⟩ + +/-- A buffer byte, read through a frame of the states. -/ +theorem buf_frame {σ : State} {m m' : Mem} {rs : List Region} + (hd : ∀ r ∈ rs, Region.Disjoint ⟨at' σ oBuf, 2720⟩ r) (hf : Frame rs m m') {k p : Nat} (hk : k < 4) + (hp' : p < 680) : m' (at' σ (oBuf + 680 * k + p)) = m (at' σ (oBuf + 680 * k + p)) := by + have := hf.bytes (R := ⟨at' σ oBuf, 2720⟩) hd (by simp only; omega) (i := 680 * k + p) (by simp only; omega) + simpa only [at', Offset.add_add, Nat.add_assoc] using this + +/-- The permutation, after `n` squeezes. -/ +theorem perm_ok {σ : State} (hp : Pre σ) {n : Nat} (hn : n < 5) {s : State} (h : SqInv σ n s) + {rest : Prog isa} {Q : State → Prop} (kont : ∀ s', Env σ s' ∧ + (∀ r < 24, ∀ k < 4, s'.mem.readW (la (scr σ) (50 + r) k) 64 = RC r) ∧ + Lanes4 s'.mem (scr σ) (fun k => iterF (n + 1) (A0 (B σ k))) ∧ + (∀ k < 4, ∀ p < 136 * n, s'.mem (at' σ (oBuf + 680 * k + p)) = hByte (B σ k) p) → WP isa rest s' Q) : + WP isa (.seq (.block permArgs) (.seq Impl.Sha3.X86_64.X4.permute4 rest)) s Q := by + refine WP.seq (WP.mono (args_ok h.env) fun s₁ ⟨⟨hm, hdi, hsi, hdx, hcx⟩, k₁⟩ => ?_) + have hrd : s₁.rd = σ.rd := k₁.2.1.trans h.env.rd + have hwr : s₁.wr = σ.wr := k₁.2.2.trans h.env.wr + refine WP.seq (WP.mono (permute4_ok (A := fun k => iterF n (A0 (B σ k))) (pre4 hp hrd hwr (by rw [hm]; exact h.rc)) + hdi hsi hdx (by rw [hcx, at', at', Offset.add_add]) (by rw [hm]; exact h.lanes)) + fun s₂ ⟨hl, hf, hrd₂, hwr₂, _, hg⟩ => kont s₂ ?_) + have hsub : ∀ r ∈ [(⟨scr σ, 800⟩ : Region), ⟨at' σ 800, 800⟩], Region.Sub r ⟨scr σ, oSave⟩ := by + intro r hr + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl + · exact Region.sub_prefix (by simp only [oSave]; omega) + · exact Offset.sub_base _ (by simp only [oSave]; omega) + refine ⟨Env.low (h.env.keep hm k₁ (by decide)) hsub hf hrd₂ hwr₂ fun r hr => hg r + (by simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide) + (by simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide), + fun r hr k hk => ?_, ?_, fun k hk p hp' => ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (by + simpa using ⟨Offset.disjoint_base (scr σ) (k := 800) (d := 32 * (50 + r) + 8 * k) (n := 8) (by omega) (by omega), + Offset.disjoint (scr σ) (d := 32 * (50 + r) + 8 * k) (n := 8) (e := 800) (k := 800) (by omega) (by omega) + (by omega)⟩) (by decide), hm] + exact h.rc r hr k hk + · intro i hi k hk + rw [hl i hi k hk] + rfl + · rw [buf_frame (by simpa using ⟨Offset.disjoint_base (scr σ) (k := 800) (d := oBuf) (n := 2720) (by simp only [oBuf]; omega) + (by simp only [oBuf]; omega), Offset.disjoint (scr σ) (d := oBuf) (n := 2720) (e := 800) (k := 800) + (by simp only [oBuf]; omega) (by simp only [oBuf]; omega) (by omega)⟩) hf hk (by omega), hm] + exact h.buf k hk p hp' + + +/-! ## Copying the output -/ + +/-- During the copy of block `n`: the first `I` lanes of state `K` copied, +and all of the states before it. -/ +structure EXI (σ : State) (n K I : Nat) (s : State) : Prop where + env : Env σ s + rc : ∀ r < 24, ∀ k < 4, s.mem.readW (la (scr σ) (50 + r) k) 64 = RC r + lanes : Lanes4 s.mem (scr σ) (fun k => iterF (n + 1) (A0 (B σ k))) + buf : ∀ k < 4, ∀ p < 680, (p < 136 * n ∨ (136 * n ≤ p ∧ p < 136 * n + 136 ∧ (k < K ∨ (k = K ∧ p < 136 * n + 8 * I)))) → + s.mem (at' σ (oBuf + 680 * k + p)) = hByte (B σ k) p + +open VG.Proof.Sha3.X86_64 (wp_movm wp_store wp_nil) in +theorem ext_step {σ : State} (hp : Pre σ) {n K I : Nat} (hn : n < 5) (hK : K < 4) (hI : I < 17) {s : State} + (h : EXI σ n K I s) : + WP isa (.block [.mov .rax (.mem (at_ .rbx (32 * I + 8 * K))), + .store (at_ .rbx (oBuf + 680 * K + 136 * n + 8 * I)) .rax]) s (EXI σ n K (I + 1)) := by + refine wp_movm (a := at' σ (32 * I + 8 * K)) (by rw [ea_at, h.env.rbx]) + (in_scr' hp h.env.rd h.env.wr (by omega)) fun s₁ u₁ => wp_store (a := at' σ (oBuf + 680 * K + 136 * n + 8 * I)) + (by rw [ea_at, u₁.other _ (by decide), h.env.rbx]) (by rw [u₁.wr]; exact in_scr hp h.env.wr (by simp only [oBuf]; omega)) + fun s₂ g₂ m₂ r₂ w₂ => wp_nil ?_ + have hm : s₂.mem = s.mem.writeW (at' σ (oBuf + 680 * K + 136 * n + 8 * I)) (s.mem.readW (at' σ (32 * I + 8 * K)) 64) := by + rw [m₂, u₁.mem, u₁.gpr] + have hf : Frame [⟨at' σ (oBuf + 680 * K + 136 * n + 8 * I), 8⟩] s.mem s₂.mem := by + rw [hm]; exact (Frame.refl _ _).writeW (List.mem_singleton_self _) _ (Region.contains_self _ _) + refine ⟨Env.low h.env (fun r hr => by + simp only [List.mem_singleton] at hr; subst hr + exact Offset.sub_base _ (by simp only [oBuf, oSave]; omega)) hf (r₂.trans u₁.rd) (w₂.trans u₁.wr) + fun r hr => by + rw [g₂, u₁.other r (by simp only [List.mem_cons, List.not_mem_nil, or_false] at hr; rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide)], + fun r hr k hk => ?_, fun i hi k hk => ?_, fun k hk p hp' hc => ?_⟩ + · have e := readW_writeW_off s.mem (scr σ) (s.mem.readW (at' σ (32 * I + 8 * K)) 64) (d := 32 * (50 + r) + 8 * k) + (e := oBuf + 680 * K + 136 * n + 8 * I) (n := 8) (by omega) (by simp only [oBuf]; omega) + (by simp only [oBuf]; omega) + rw [hm]; exact e.trans (h.rc r hr k hk) + · have e := readW_writeW_off s.mem (scr σ) (s.mem.readW (at' σ (32 * I + 8 * K)) 64) (d := 32 * i + 8 * k) + (e := oBuf + 680 * K + 136 * n + 8 * I) (n := 8) (by omega) (by simp only [oBuf]; omega) + (by simp only [oBuf]; omega) + rw [hm]; exact e.trans (h.lanes i hi k hk) + · rw [hm] + by_cases hw : k = K ∧ 136 * n + 8 * I ≤ p ∧ p < 136 * n + 8 * I + 8 + · obtain ⟨rfl, h₁, h₂⟩ := hw + rw [wb_in _ _ _ (by omega) (by simp only [oBuf]; omega) (by decide), + show 8 * (oBuf + 680 * k + p - (oBuf + 680 * k + 136 * n + 8 * I)) = 8 * (p - 136 * n - 8 * I) by omega, + byte_readW _ _ (by omega), at', Offset.add_add, + show 32 * I + 8 * k + (p - 136 * n - 8 * I) = 32 * ((8 * I + (p - 136 * n - 8 * I)) / 8) + 8 * k + + (8 * I + (p - 136 * n - 8 * I)) % 8 by omega, + byte_of_lanes4 h.lanes hk (by omega), ← hByte_A0 (B_length σ k) (by omega), + show 136 * n + (8 * I + (p - 136 * n - 8 * I)) = p by omega] + · rw [wb_out _ _ _ (by simp only [oBuf]; omega) (by simp only [oBuf]; omega) (by simp only [oBuf]; omega)] + exact h.buf k hk p hp' (by omega) + +theorem extract_eq (n : Nat) : extract n = (List.range 4).flatMap fun K => (List.range 17).flatMap fun I => + [.mov .rax (.mem (at_ .rbx (32 * I + 8 * K))), .store (at_ .rbx (oBuf + 680 * K + 136 * n + 8 * I)) .rax] := rfl + +/-- Block `n` of each state's output. -/ +theorem extract_ok {σ : State} (hp : Pre σ) {n : Nat} (hn : n < 5) {s : State} (h : EXI σ n 0 0 s) : + WP isa (.block (extract n)) s (SqInv σ (n + 1)) := by + rw [extract_eq] + refine WP.mono (wp_range_flatMap (M := isa) (fun K s => EXI σ n K 0 s) (fun K s hK h => ?_) 4 (Nat.le_refl _) s h) + fun s' h' => ⟨h'.env, h'.rc, h'.lanes, fun k hk p hp' => h'.buf k hk p (by omega) (by omega)⟩ + refine WP.mono (wp_range_flatMap (M := isa) (fun I s => EXI σ n K I s) (fun I s hI h => ext_step hp hn hK hI h) + 17 (Nat.le_refl _) s h) fun s' h' => ⟨h'.env, h'.rc, h'.lanes, fun k hk p hp' hc => h'.buf k hk p hp' (by omega)⟩ + +/-- `squeeze4 n`: after `n + 1` squeezes. -/ +theorem sq_ok {σ : State} (hp : Pre σ) {n : Nat} (hn : n < 5) {s : State} (h : SqInv σ n s) : + WP isa (squeeze4 n) s (SqInv σ (n + 1)) := by + unfold squeeze4 + exact perm_ok hp hn h fun s' ⟨he, hrc, hl, hb⟩ => + extract_ok hp hn ⟨he, hrc, hl, fun k hk p _ hc => hb k hk p (by omega)⟩ + +end VG.Proof.MlDsa.X86_64.Mask4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Top.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Top.lean new file mode 100644 index 000000000..908739367 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Top.lean @@ -0,0 +1,220 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.M4Squeeze + +/-! +# ML-DSA on x86-64: `vg_mldsa_expand_mask_poly4_avx2`, correctness + +Untrusted: everything here is checked by Lean. The pieces, in order: the +prologue, the round constants and the padded seeds (`M4Absorb.lean`), five +squeezes (`M4Squeeze.lean`), which leave the first 680 bytes of SHAKE256 of +each seed in its buffer, the branch on `γ₁`, the four unpackings, each the +loop of `vg_mldsa_expand_mask_poly` (`emBody_ok`) on the output of a seed +(`unpack_ok`), and the epilogue. +-/ + +namespace VG.Proof.MlDsa.X86_64.Mask4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlDsa.X86_64.Sample.Mask4 +open VG.Impl.MlKem.X86_64.Sample4 (oRc oBuf) +open VG.Impl.MlDsa.X86_64.Sample (emBody) +open VG.Proof.MlKem.X86_64 +open VG.Proof.MlDsa.X86_64.Sample (GPre emBody_ok emOk emV gOf) +open VG.Proof.MlDsa.Sample (emC emC_eq expandMask_getElem polyIs_of_coeffAt) +open VG.Spec.MlDsa (H coeffAt) +open VG.Proof.MlDsa.Sample (coeffAddr) +open VG.Proof.Sha3.X86_64.X4 (la) + +/-- The first 640 bytes of SHAKE256 of seed `k`. -/ +abbrev X (σ : State) (k : Nat) : List Byte := H (B σ k) 640 + +/-- After the squeezes, and the unpackings of the first `K` seeds, `c` bits a coefficient. -/ +structure UI (σ : State) (c K : Nat) (s : State) : Prop where + env : Env σ s + buf : ∀ k < 4, ∀ p < 680, s.mem (at' σ (oBuf + 680 * k + p)) = hByte (B σ k) p + st : ∀ k < K, ∀ i < 256, coeffAt s.mem (poly4 (aP σ) k) i = emV (X σ k) c i + +/-- During the unpacking of seed `K`: `g` groups of four. -/ +structure EI (σ : State) (c K g : Nat) (s : State) : Prop where + ui : UI σ c K s + rsi : s.gpr .rsi = at' σ (oBuf + 680 * K) + BitVec.ofNat 64 (c / 2 * g) + rdi : s.gpr .rdi = poly4 (aP σ) K + BitVec.ofNat 64 (16 * g) + cur : ∀ i < 4 * g, coeffAt s.mem (poly4 (aP σ) K) i = emV (X σ K) c i + +section +variable {σ : State} (hp : Pre σ) +include hp + +/-- The prologue, the round constants and the padded seeds. -/ +theorem start_ok : WP isa (.block (pro ++ Impl.Sha3.X86_64.X4.rcTable .rbx (oRc / 32) ++ absorb4)) σ (SqInv σ 0) := by + rw [WP.block_append_iff, WP.block_append_iff] + refine WP.mono (pro_ok hp) fun s₁ h₁ => WP.mono (rc_ok hp h₁) fun s₂ h₂ => ?_ + have he₂ : Env σ s₂ := Env.low h₁ (rs := [⟨at' σ oRc, 768⟩]) (fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact Offset.sub_base _ (by simp only [oRc, oSave]; omega)) + h₂.frame h₂.keep.2.1 h₂.keep.2.2 fun r hr => h₂.keep.gpr (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide) + refine WP.mono (absorb_ok hp (m₁ := s₂.mem) ⟨he₂, Frame.refl _ _⟩) + fun s₃ ⟨a₃, b₃⟩ => ⟨a₃.env, fun r hr k hk => ?_, lanes_A0 b₃, fun _ _ p hp' => absurd hp' (by omega)⟩ + rw [a₃.frame.readW (Region.contains_self _ _) (by + simpa using Offset.disjoint_base (scr σ) (k := 800) (d := 32 * (50 + r) + 8 * k) (n := 8) (by omega) (by omega)) + (by decide)] + exact h₂.rc r hr k hk + +/-- `Env` after writes to polynomial `K`. -/ +theorem Env.poly {K : Nat} (hK : K < 4) {s s' : State} (he : Env σ s) + (hf : Frame [⟨poly4 (aP σ) K, 1024⟩] s.mem s'.mem) (hrd : s'.rd = s.rd) (hwr : s'.wr = s.wr) + (hg : ∀ r ∈ [Reg.rbx, .r12, .r13, .r14, .rsp, .r15], s'.gpr r = s.gpr r) : Env σ s' := by + refine ⟨hrd.trans he.rd, hwr.trans he.wr, by rw [hg .rbx (by simp), he.rbx], by rw [hg .r12 (by simp), he.r12], + by rw [hg .r13 (by simp), he.r13], by rw [hg .r14 (by simp), he.r14], by rw [hg .rsp (by simp), he.rsp], + by rw [hg .r15 (by simp), he.r15], fun i hi => ?_, ?_⟩ + · rw [hf.readW (Region.contains_self _ _) (by + simpa using ((hp.a_scr.sub_left (sub_poly hK)).sub_right (sub_scr (σ := σ) (a := oSave + 8 * i) (n := 8) + (by simp only [oSave]; omega))).symm) (by decide)] + exact he.saved i hi + · exact he.frame.trans (hf.sub fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact ⟨aR σ, by simp, sub_poly hK⟩) + +/-- The facts a group of the unpacking of seed `K` needs. -/ +theorem gpre {c K g : Nat} (hK : K < 4) (hg : g < 64) {s : State} (h : EI σ c K g s) : + GPre c (X σ K) (at' σ (oBuf + 680 * K)) (poly4 (aP σ) K) g s := by + refine ⟨h.rsi, h.rdi, hg, fun j hj => ?_, fun j hj => ?_, fun i hi => ?_, fun j hj hc => ?_⟩ + · rw [at', Offset.add_add, ← at', h.ui.buf K hK j (by omega)]; exact (H_getD _ hj).symm + · rw [at', Offset.add_add, ← at'] + exact in_scr' hp h.ui.env.rd h.ui.env.wr (by simp only [oBuf]; omega) + · rw [h.ui.env.wr, hp.wr] + refine ⟨aR σ, by simp, ?_⟩ + rw [coeffAddr, poly4, Offset.add_add] + exact Offset.contains_base _ (by omega) (by omega) + · rw [at', Offset.add_add] at hc + exact (hp.a_scr.sub_left (sub_poly hK)) _ hc + (Offset.contains_base _ (by simp only [oBuf]; omega) (by simp only [oBuf]; omega)) + +/-- An iteration of the unpacking of seed `K`. -/ +theorem ei_step {c K g : Nat} (hc : emOk c) (hK : K < 4) (hg : g < 64) {s : State} (h : EI σ c K g s) : + WP isa (.block (emBody c)) s fun s' => EI σ c K (g + 1) s' ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ + s'.zf = some (s.gpr .rcx - 1 == 0) := by + refine WP.mono (emBody_ok hc (gpre hp hK hg h)) fun s' ⟨hk, hf, hst, hsame, hsi, hdi, hcx, hz⟩ => ?_ + refine ⟨⟨⟨Env.poly hp hK h.ui.env hf hk.2.1 hk.2.2 fun r hr => hk.gpr (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide), fun k hk' p hp' => ?_, fun k hk' i hi => ?_⟩, + hsi, hdi, fun i hi => ?_⟩, hcx, hz⟩ + · rw [buf_frame (by simpa using ((hp.a_scr.sub_left (sub_poly hK)).sub_right (sub_scr (σ := σ) (a := oBuf) + (n := 2720) (by simp only [oBuf]; omega))).symm) hf hk' hp'] + exact h.ui.buf k hk' p hp' + · rw [show coeffAt s'.mem (poly4 (aP σ) k) i = coeffAt s.mem (poly4 (aP σ) k) i from + hf.readW (VG.Proof.MlDsa.Sample.coeff_contains _ hi) (fun r hr => by + simp only [List.mem_singleton] at hr; subst hr + have hd := Offset.disjoint (aP σ) (d := 1024 * k) (n := 1024) (e := 1024 * K) (k := 1024) (by omega) + (by omega) (by omega) + simpa [poly4] using hd) (by decide)] + exact h.ui.st k hk' i hi + · by_cases hlo : i < 4 * g + · rw [hsame i (by omega) (.inl hlo)]; exact h.cur i hlo + · have := hst (i - 4 * g) (by omega) + rwa [show 4 * g + (i - 4 * g) = i by omega] at this + +/-- The unpacking of seed `K`. -/ +theorem unpack_ok {c K : Nat} (hc : emOk c) (hK : K < 4) {s : State} (h : UI σ c K s) : + WP isa (unpack c K) s (UI σ c (K + 1)) := by + unfold unpack + refine WP.seq (WP.mono (WP.keep [.rsi, .rdi, .rcx] (Q := fun s' => s'.mem = s.mem ∧ + s'.gpr .rsi = at' σ (oBuf + 680 * K) ∧ s'.gpr .rdi = poly4 (aP σ) K ∧ s'.gpr .rcx = BitVec.ofNat 64 64) + (by xrun [h.env.rbx, h.env.r13, sx_ofNat (show oBuf + 680 * K < 2 ^ 31 by simp only [oBuf]; omega), + sx_ofNat (show 1024 * K < 2 ^ 31 by omega)]) (by rfl)) fun s1 ⟨⟨hm1, hsi1, hdi1, hcx1⟩, k1⟩ => ?_) + have he1 : Env σ s1 := Env.low h.env (rs := []) (by simp) (by rw [hm1]; exact Frame.refl _ _) k1.2.1 k1.2.2 + fun r hr => k1.gpr (by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl <;> decide) + refine wp_countdown (N := 64) (by decide) (by decide) (EI σ c K) (fun g hg s hI _ => + WP.mono (ei_step hp hc hK hg hI) fun s' ⟨hI', hc', hz⟩ => ⟨hI', hc', hz⟩) (fun s' hI => + ⟨hI.ui.env, hI.ui.buf, fun k hk i hi => ?_⟩) + ⟨⟨he1, fun k hk p hp' => by rw [hm1]; exact h.buf k hk p hp', fun k hk i hi => by rw [hm1]; exact h.st k hk i hi⟩, + by rw [hsi1]; simp, by rw [hdi1]; simp, fun i hi => absurd hi (by omega)⟩ hcx1 + by_cases e : k = K + · subst e; exact hI.cur i (by omega) + · exact hI.ui.st k (by omega) i hi + +/-- The four unpackings. -/ +theorem unpack4_ok {c : Nat} (hc : emOk c) {s : State} (h : UI σ c 0 s) : WP isa (unpack4 c) s (UI σ c 4) := + WP.seq (WP.mono (unpack_ok hp hc (by decide) h) fun _ h₁ => WP.seq (WP.mono (unpack_ok hp hc (by decide) h₁) + fun _ h₂ => WP.seq (WP.mono (unpack_ok hp hc (by decide) h₂) fun _ h₃ => unpack_ok hp hc (by decide) h₃))) + +omit hp in +theorem sx17 : BitVec.signExtend 64 (0x20000 : BitVec 32) = BitVec.ofNat 64 0x20000 := by decide + +/-- The branch on `γ₁`, and the unpackings for it. -/ +theorem sel_ok {s : State} (h : SqInv σ 5 s) {Q : State → Prop} (kont : ∀ s', UI σ (emC (gOf σ)) 4 s' → + WP isa (.block epi) s' Q) : + WP isa (.seq (.block [.vop .vzeroupper, .alu32 .cmp .r14 (.imm 0x20000)]) + (.seq (.ite .e (unpack4 18) (unpack4 20)) (.block epi))) s Q := by + refine WP.seq (WP.mono (WP.keep [.r14] (Q := fun s' => s'.mem = s.mem ∧ s'.gpr .r14 = s.gpr .r14 ∧ + s'.zf = some (BitVec.setWidth 32 (s.gpr .r14) - 0x20000 == 0)) (by xrun; exact ⟨rfl, rfl, rfl⟩) (by decide)) + fun s1 ⟨⟨hm1, h14, hz1⟩, k1'⟩ => ?_) + have k1 : Keep [] s s1 := ⟨fun r _ => by + by_cases e : r = .r14 + · subst e; exact h14 + · exact k1'.gpr (by simp [e]), k1'.2⟩ + have hr14 : BitVec.setWidth 32 (s.gpr .r14) = BitVec.ofNat 32 (gOf σ) := by + rw [h.env.r14, gOf, BitVec.ofNat_toNat, BitVec.setWidth_eq] + have hU : ∀ c, UI σ c 0 s1 := fun c => ⟨Env.low h.env (rs := []) (by simp) (by rw [hm1]; exact Frame.refl _ _) + k1.2.1 k1.2.2 fun r _ => k1.gpr (List.not_mem_nil), + fun k hk p hp' => by rw [hm1]; exact h.buf k hk p (by omega), fun _ hk => absurd hk (by omega)⟩ + refine WP.seq (WP.mono (?_ : WP isa (.ite .e (unpack4 18) (unpack4 20)) s1 (UI σ (emC (gOf σ)) 4)) kont) + rcases hp.gamma with he | he + · refine WP.ite true (by show s1.zf = _; rw [hz1, hr14, he]; rfl) (fun _ => ?_) (fun hb => absurd hb (by decide)) + rw [show emC (gOf σ) = 18 by rw [he]; rfl]; exact unpack4_ok hp (.inl rfl) (hU 18) + · refine WP.ite false (by show s1.zf = _; rw [hz1, hr14, he]; rfl) (fun hb => absurd hb (by decide)) (fun _ => ?_) + rw [show emC (gOf σ) = 20 by rw [he]; rfl]; exact unpack4_ok hp (.inr rfl) (hU 20) + +omit hp in +theorem epi_eq : epi = [.mov .r14 (.mem (at_ .rbx 5120)), .mov .r13 (.mem (at_ .rbx 5112)), + .mov .r12 (.mem (at_ .rbx 5104)), .mov .rbp (.mem (at_ .rbx 5096)), .mov .rbx (.mem (at_ .rbx 5088))] := rfl + +/-- The postcondition, and the callee-saved registers restored. -/ +theorem end_ok {s : State} (h : UI σ (emC (gOf σ)) 4 s) : + WP isa (.block epi) s fun s' => em4K.post σ s' ∧ gprPreserved σ s' := by + have hin : ∀ i < 5, InRegions (s.rd ++ s.wr) (scr σ + BitVec.ofNat 64 (oSave + 8 * i)) 8 := fun i hi => + in_scr' hp h.env.rd h.env.wr (by simp only [oSave]; omega) + rw [epi_eq] + refine WP.mono (WP.keep [.r14, .r13, .r12, .rbp, .rbx] (Q := fun s' => s'.mem = s.mem ∧ + s'.gpr .r14 = s.mem.readW (at' σ (oSave + 8 * 4)) 64 ∧ s'.gpr .r13 = s.mem.readW (at' σ (oSave + 8 * 3)) 64 ∧ + s'.gpr .r12 = s.mem.readW (at' σ (oSave + 8 * 2)) 64 ∧ s'.gpr .rbp = s.mem.readW (at' σ (oSave + 8 * 1)) 64 ∧ + s'.gpr .rbx = s.mem.readW (at' σ (oSave + 8 * 0)) 64) + (by + have h0 := hin 0 (by decide); have h1 := hin 1 (by decide); have h2 := hin 2 (by decide) + have h3 := hin 3 (by decide); have h4 := hin 4 (by decide) + simp only [oSave, Nat.reduceMul, Nat.reduceAdd] at h0 h1 h2 h3 h4 + xrun [h0, h1, h2, h3, h4, h.env.rbx] + exact ⟨rfl, rfl, rfl, rfl, rfl⟩) + (by decide)) fun s' ⟨⟨hm, h14, h13, h12, hbp, hbx⟩, k⟩ => ?_ + obtain ⟨_, _, hγ⟩ := emC_eq hp.gamma + refine ⟨fun K hK => polyIs_of_coeffAt fun i hi => ?_, fun r hr => ?_, ?_⟩ + · rw [expandMask_getElem _ hp.gamma hi, hm, h.st K hK i (by omega)] + simp only [emV] + congr 3 + rw [← hγ] + · simp only [calleeSaved, List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl | rfl | rfl + · rw [hbx]; exact h.env.saved 0 (by decide) + · rw [hbp]; exact h.env.saved 1 (by decide) + · rw [k.gpr (by decide)]; exact h.env.rsp + · rw [h12]; exact h.env.saved 2 (by decide) + · rw [h13]; exact h.env.saved 3 (by decide) + · rw [h14]; exact h.env.saved 4 (by decide) + · rw [k.gpr (by decide)]; exact h.env.r15 + · rw [hm] + exact h.env.frame.readW (Region.contains_self _ _) (by + simpa using ⟨hp.ret_a, hp.ret_scr, Offset.base_disjoint_below (σ.gpr .rsp) (n := 24) (k := 8) (by omega)⟩) + (by decide) + +end + +theorem correct (σ : State) (hs : em4K.pre σ) : + ∃ t s', Exec isa expandMask4Avx2 σ t s' ∧ abiPreserved σ s' ∧ em4K.post σ s' := by + have hp := pre_of hs + obtain ⟨t, s', he, hF⟩ := WP.seq (WP.mono (start_ok hp) fun _ h₀ => + WP.seq (WP.mono (sq_ok hp (by decide) h₀) fun _ h₁ => WP.seq (WP.mono (sq_ok hp (by decide) h₁) fun _ h₂ => + WP.seq (WP.mono (sq_ok hp (by decide) h₂) fun _ h₃ => WP.seq (WP.mono (sq_ok hp (by decide) h₃) fun _ h₄ => + WP.seq (WP.mono (sq_ok hp (by decide) h₄) fun _ h₅ => sel_ok hp h₅ fun _ hu => end_ok hp hu)))))) + exact ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he hF.2, hF.1⟩ + +end VG.Proof.MlDsa.X86_64.Mask4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Verified.lean new file mode 100644 index 000000000..34a82c12c --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sample/M4Verified.lean @@ -0,0 +1,102 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Sample.M4Scalar + +/-! +# ML-DSA on x86-64: `vg_mldsa_expand_mask_poly4` and `vg_mldsa_expand_mask_poly4_avx2`, verified + +Untrusted: everything here is checked by Lean. The baseline implementation +is constant time as the calls are (`expandMask_ct`, from their pointers and +`γ₁`), and the contract of the proofs (`em4K`) is the shared one of `Spec/`. +-/ + +namespace VG.Proof.MlDsa.X86_64.Mask4 + +open VG VG.X86_64 VG.Impl.MlKem.X86_64 VG.Impl.MlDsa.X86_64.Sample.Mask4 +open VG.Proof.MlKem.X86_64 +open VG.Proof.MlDsa.X86_64.Sample (emK gOf expandMask_correct expandMask_ct) + +/-- The call on seed `K`. -/ +theorem call_ct {K : Nat} (hK : K < 4) : + RelCT isa (R fun σ s => ArgI σ K s) (.call "vg_mldsa_expand_mask_poly" Impl.MlDsa.X86_64.Sample.expandMask) + (R fun σ s => PC σ (K + 1) s) := + relInv (fun σ s hp h => callK_ok (pre_of hp) hK h) (RelCT.callEx expandMask_correct expandMask_ct + fun s₁ s₂ ⟨σ₁, σ₂, p₁, p₂, hq, h₁, h₂⟩ => by + have hsp : s₁.gpr .rsp = s₂.gpr .rsp := by rw [h₁.pinv.env.rsp, h₂.pinv.env.rsp, hq.2.2.2.2] + refine ⟨_, _, _, _, argK_pre (pre_of p₁) hK h₁, argK_pre (pre_of p₂) hK h₂, ?_, + (cov (pre_of p₁) h₁.pinv.env hK).1, (cov (pre_of p₁) h₁.pinv.env hK).2, + (cov (pre_of p₂) h₂.pinv.env hK).1, (cov (pre_of p₂) h₂.pinv.env hK).2, hsp⟩ + simp only [emK, State.withRegions_gpr, State.callEntry_rsp, + ce_gpr' _ (by decide : Reg.rdi ≠ .rsp), ce_gpr' _ (by decide : Reg.rsi ≠ .rsp), + ce_gpr' _ (by decide : Reg.rdx ≠ .rsp), ce_gpr' _ (by decide : Reg.rcx ≠ .rsp), h₁.rdi, h₂.rdi, h₁.rsi, h₂.rsi, + h₁.rdx, h₂.rdx, h₁.rcx, h₂.rcx] + simp only [sd, aP, at', scr, hq.1, hq.2.1, hq.2.2.1, hq.2.2.2.1, hsp, and_self]) + +/-- The call on seed `K`, given the taint analysis of its arguments. -/ +theorem callK_ct {K : Nat} (hK : K < 4) {hc : VG.Taint.Hint X86_64.Taint.T} + (c : (taint.check (X86_64.Taint.ofRegs [.r12, .r13, .rbx]) + (.block [.mov .rdi (.reg .r12), .alu .add .rdi (.imm (BitVec.ofNat 32 (66 * K))), .mov .rsi (.reg .r14), + .mov .rdx (.reg .r13), .alu .add .rdx (.imm (BitVec.ofNat 32 (1024 * K))), .mov .rcx (.reg .rbx), + .alu .add .rcx (.imm (BitVec.ofNat 32 oScalar))]) hc).isSome = true) : + RelCT isa (R fun σ s => PC σ K s) (callK K) (R fun σ s => PC σ (K + 1) s) := + RelCT.seq (relInv (fun σ s _ h => argsK_ok hK h) (taintRel [.r12, .r13, .rbx] + (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl + · rw [h₁.env.r12, h₂.env.r12, sd, sd, hq.1] + · exact env_r13 hq h₁.env h₂.env + · exact env_rbx hq h₁.env h₂.env) c)) + (call_ct hK) + +theorem ct_scalar : ConstantTime isa em4K.pre em4K.pub expandMask4 := by + refine relStart (Q := fun _ _ => True) (RelCT.seq (relInv (I' := fun σ s => PC σ 0 s) + (fun σ s hp h => by + subst h + exact WP.mono (pro_ok (pre_of hp)) fun _ h => ⟨h, fun _ h => absurd h (by omega)⟩) + (taintRel [.rdi, .rdx, .rcx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + subst h₁ h₂ + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl + exacts [hq.1, hq.2.2.1, hq.2.2.2.1]) (by taint_decide))) ?_) + refine RelCT.seq (callK_ct (K := 0) (by decide) (by taint_decide)) (RelCT.seq (callK_ct (K := 1) (by decide) + (by taint_decide)) (RelCT.seq (callK_ct (K := 2) (by decide) (by taint_decide)) + (RelCT.seq (callK_ct (K := 3) (by decide) (by taint_decide)) ?_))) + exact taintRel [.rbx] (fun x y ⟨σ₁, σ₂, _, _, hq, h₁, h₂⟩ r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact env_rbx hq h₁.env h₂.env) (by taint_decide) + +/-- A state satisfying the precondition. -/ +def em4Sat : State where + gpr r := match r with + | .rdi => 0x1000 | .rsi => 0x20000 | .rdx => 0x2000 | .rcx => 0x4000 | .rsp => 0x8000 | _ => 0 + cf := none + zf := none + sf := none + of := none + mem _ := 0 + rd := [⟨0x1000, 264⟩] + wr := [⟨0x2000, 4096⟩, ⟨0x4000, 8192⟩] + +theorem em4_verified (c : Prog isa) + (hc : ∀ σ, em4K.pre σ → ∃ t s', Exec isa c σ t s' ∧ abiPreserved σ s' ∧ em4K.post σ s') + (ht : ConstantTime isa em4K.pre em4K.pub c) : + Verified X86_64.target c (Spec.MlDsa.expandMask4Contract X86_64.abi 24) := + Verified.of_correct hc ht + { pre := by sig_implies_pre [Spec.MlDsa.expandMask4Contract, Spec.MlDsa.expandMask4Sig, em4K, X86_64.abi, + X86_64.argRegs] + post := by + intro s s' _ h + sig_post [Spec.MlDsa.expandMask4Contract, Spec.MlDsa.expandMask4Sig, em4K, X86_64.abi, X86_64.argRegs] + exact h + pub := by + intro s₁ s₂ _ _ h + sig_pub [Spec.MlDsa.expandMask4Contract, Spec.MlDsa.expandMask4Sig, em4K, X86_64.abi, X86_64.argRegs] at h + obtain ⟨hsp, hdi, hsi, hdx, hcx⟩ := h + exact ⟨hdi, hsi, hdx, hcx, hsp⟩ + sat := by sig_implies_sat [Spec.MlDsa.expandMask4Contract, Spec.MlDsa.expandMask4Sig, em4K, X86_64.abi, + X86_64.argRegs] [em4Sat] using em4Sat } + +theorem expandMask4Avx2_verified : Verified X86_64.target expandMask4Avx2 + (Spec.MlDsa.expandMask4Contract X86_64.abi 24) := em4_verified _ correct ct + +theorem expandMask4_verified : Verified X86_64.target expandMask4 + (Spec.MlDsa.expandMask4Contract X86_64.abi 24) := em4_verified _ correct_scalar ct_scalar + +end VG.Proof.MlDsa.X86_64.Mask4 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean index fc33ea67e..bbe9041e2 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean @@ -53,6 +53,7 @@ def prims : Prims where bitUnpack := Impl.MlDsa.X86_64.Pack.bitUnpack hintBitPack := Impl.MlDsa.X86_64.Pack.hintBitPack rej4 := Impl.MlDsa.X86_64.Sample.Rej4.rejNTT4 + expandMask4 := Impl.MlDsa.X86_64.Sample.Mask4.expandMask4 /-- The primitives, with the polynomial arithmetic of `B`. -/ def primsWith (B : Impl.MlDsa.X86_64.Arith.Backend) : Prims := @@ -64,6 +65,7 @@ def primsWith (B : Impl.MlDsa.X86_64.Arith.Backend) : Prims := add := B.add sub := B.sub rej4 := B.rej4 + expandMask4 := B.expandMask4 highBits := B.highBits lowBits := B.lowBits normLt := B.normLt @@ -173,6 +175,10 @@ def prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) signStack where have := v.ok.rej4.depth show 8 * (v.code.rej4.depth + 1) ≤ signStack unfold signStack; omega⟩ + expandMask4 := ⟨24, by decide, v.ok.expandMask4.ver, v.ok.expandMask4.nosp, by + have := v.ok.expandMask4.depth + show 8 * (v.code.expandMask4.depth + 1) ≤ signStack + unfold signStack; omega⟩ rej4Ret := fun s₁ s₂ t₁ t₂ s₁' s₂' ⟨h₁, h₂, hp⟩ e₁ e₂ => ⟨v.ok.rej4.ver.2.1 s₁ s₂ t₁ t₂ s₁' s₂' h₁ h₂ hp e₁ e₂, show _ = _ by diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseC.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseC.lean index 21a3f4677..e1f612d07 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseC.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseC.lean @@ -87,16 +87,16 @@ theorem IL.st {p : Params} {D : Nat} {σ s : State} {t : Nat} (h : IL p D σ t s /-! ## `κ + r` -/ -theorem setKappa_ok (r : Nat) (hr : r < 2 ^ 31) (s : State) - (h1 : InRegions (s.rd ++ s.wr) (pa s (sc oKAP)) 8) (h2 : InRegions s.wr (pa s (sc (oMS + 64))) 1) - (h3 : InRegions s.wr (pa s (sc (oMS + 65))) 1) : - WP isa (.block (setKappa r)) s fun s' => - s'.mem = (s.mem.writeW (pa s (sc (oMS + 64))) +theorem setKap_ok (o r : Nat) (hr : r < 2 ^ 31) (s : State) + (h1 : InRegions (s.rd ++ s.wr) (pa s (sc oKAP)) 8) (h2 : InRegions s.wr (pa s (sc o)) 1) + (h3 : InRegions s.wr (pa s (sc (o + 1))) 1) : + WP isa (.block (setKap o r)) s fun s' => + s'.mem = (s.mem.writeW (pa s (sc o)) ((s.mem.readW (pa s (sc oKAP)) 64 + BitVec.ofNat 64 r).setWidth 8)).writeW - (pa s (sc (oMS + 65))) (((s.mem.readW (pa s (sc oKAP)) 64 + BitVec.ofNat 64 r) >>> 8).setWidth 8) ∧ + (pa s (sc (o + 1))) (((s.mem.readW (pa s (sc oKAP)) 64 + BitVec.ofNat 64 r) >>> 8).setWidth 8) ∧ Keep [.rax] s s' := by refine WP.keep [.rax] ?_ (by rfl) - unfold setKappa + unfold setKap xrun [h1, h2, h3, sx_ofNat hr] theorem integerToBytes_two (x : Nat) : integerToBytes x 2 = [BitVec.ofNat 8 x, BitVec.ofNat 8 (x / 256)] := by @@ -130,32 +130,41 @@ theorem bytes2_write (m : Mem) (a : Addr) (v w : Byte) : rw [e0, e1, VG.Proof.MlKem.writeW8_apply, VG.Proof.MlKem.writeW8_apply, ifn (add_one_ne a), ifp rfl, VG.Proof.MlKem.writeW8_apply, ifp rfl] -/-- `κ + r` to `MS + 64`, as the two bytes of `ExpandMask`'s seed. -/ -theorem setKappa_okB {D : Nat} {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay D rbs wbs s) {r x : Nat} - (hr : r < 2 ^ 31) (hx : x + r < 2 ^ 16) (h1 : inB (rbs ++ wbs) (sc oKAP) 8 = true) - (h2 : inB wbs (sc (oMS + 64)) 2 = true) (hk : s.mem.readW (pa s (sc oKAP)) 64 = BitVec.ofNat 64 x) : - WP isa (.block (setKappa r)) s fun s' => PPostB D s s' [(sc (oMS + 64), 2)] ∧ +/-- `κ + r` to `scratch + o`, as the two bytes of `ExpandMask`'s seed. -/ +theorem setKap_okB {D : Nat} {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay D rbs wbs s) {r x : Nat} + {o : Nat} (hr : r < 2 ^ 31) (hx : x + r < 2 ^ 16) (h1 : inB (rbs ++ wbs) (sc oKAP) 8 = true) + (h2 : inB wbs (sc o) 2 = true) (hk : s.mem.readW (pa s (sc oKAP)) 64 = BitVec.ofNat 64 x) : + WP isa (.block (setKap o r)) s fun s' => PPostB D s s' [(sc o, 2)] ∧ (∀ r ∈ calleeSaved, s'.gpr r = s.gpr r) ∧ - bytesAt s'.mem (pa s (sc (oMS + 64))) 2 = integerToBytes (x + r) 2 := by - have e65 : pa s (sc (oMS + 65)) = pa s (sc (oMS + 64)) + 1 := (pa_sc_add s (oMS + 64) 1).symm + bytesAt s'.mem (pa s (sc o)) 2 = integerToBytes (x + r) 2 := by + have e65 : pa s (sc (o + 1)) = pa s (sc o) + 1 := (pa_sc_add s o 1).symm have w2 := L.iW h2 - have c0 : (⟨pa s (sc (oMS + 64)), 2⟩ : Region).Contains (pa s (sc (oMS + 64))) 1 := by - have := contains_offset' (base := pa s (sc (oMS + 64))) (off := 0) (len := 1) (n := 2) (by omega) (by decide) + have c0 : (⟨pa s (sc o), 2⟩ : Region).Contains (pa s (sc o)) 1 := by + have := contains_offset' (base := pa s (sc o)) (off := 0) (len := 1) (n := 2) (by omega) (by decide) rwa [BitVec.add_zero] at this - have c1 : (⟨pa s (sc (oMS + 64)), 2⟩ : Region).Contains (pa s (sc (oMS + 65))) 1 := by + have c1 : (⟨pa s (sc o), 2⟩ : Region).Contains (pa s (sc (o + 1))) 1 := by rw [e65]; exact contains_offset' (off := 1) (by omega) (by decide) - have i0 : InRegions s.wr (pa s (sc (oMS + 64))) 1 := by + have i0 : InRegions s.wr (pa s (sc o)) 1 := by have := inRegions_sub (off := 0) (l := 1) w2 (by omega) (by decide) rwa [BitVec.add_zero] at this - have i1 : InRegions s.wr (pa s (sc (oMS + 65))) 1 := by + have i1 : InRegions s.wr (pa s (sc (o + 1))) 1 := by rw [e65]; exact inRegions_sub (off := 1) (l := 1) w2 (by omega) (by decide) - refine WP.mono (setKappa_ok r hr s (L.iR h1) i0 i1) fun s' ⟨hm, k⟩ => ?_ - have hf : Frame [⟨pa s (sc (oMS + 64)), 2⟩] s.mem s'.mem := by + refine WP.mono (setKap_ok o r hr s (L.iR h1) i0 i1) fun s' ⟨hm, k⟩ => ?_ + have hf : Frame [⟨pa s (sc o), 2⟩] s.mem s'.mem := by rw [hm] exact ((Frame.refl _ _).writeW (List.mem_singleton_self _) _ c0).writeW (List.mem_singleton_self _) _ c1 refine ⟨(postB_of_keep (D := D) k (by decide) hf).1, (postB_of_keep (D := D) k (by decide) hf).2, ?_⟩ rw [hm, e65, bytes2_write, hk, ofNat64_add, kappa_bytes hx] +/-- `κ + r` to `MS + 64`, as the two bytes of `ExpandMask`'s seed. -/ +theorem setKappa_okB {D : Nat} {rbs wbs : List (Reg × Nat)} {s : State} (L : Lay D rbs wbs s) {r x : Nat} + (hr : r < 2 ^ 31) (hx : x + r < 2 ^ 16) (h1 : inB (rbs ++ wbs) (sc oKAP) 8 = true) + (h2 : inB wbs (sc (oMS + 64)) 2 = true) (hk : s.mem.readW (pa s (sc oKAP)) 64 = BitVec.ofNat 64 x) : + WP isa (.block (setKappa r)) s fun s' => PPostB D s s' [(sc (oMS + 64), 2)] ∧ + (∀ r ∈ calleeSaved, s'.gpr r = s.gpr r) ∧ + bytesAt s'.mem (pa s (sc (oMS + 64))) 2 = integerToBytes (x + r) 2 := + setKap_okB L hr hx h1 h2 hk + /-! ## `y` and `ŷ` -/ /-- Iteration `t`, with the first `r` polynomials of `y` and `ŷ`. -/ @@ -218,6 +227,134 @@ theorem maskR_ok {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} {σ : Sta rw [hP4.pa (pS_bases _), hyh3.2] at * exact hq4 +/-! ## `y` and `ŷ`, four at a time -/ + +/-- Iteration `t`, with the first `ry` polynomials of `y` and the first `r` of `ŷ`. -/ +structure ICy (p : Params) (D : Nat) (σ : State) (t ry r : Nat) (s : State) : Prop where + l : IL p D σ t s + y : Fam s (yBase p) ry (Yv p σ (p.ℓ * t)) + yh : Fam s (yhBase p) r (YHv p σ (p.ℓ * t)) + +def icyChk (p : Params) (ws : List (Ptr × Nat)) (ry r : Nat) : Bool := + ilChk p ws && famChk (sgB p) ws (yBase p) ry && famChk (sgB p) ws (yhBase p) r + +theorem ICy.step {p : Params} {D : Nat} {σ s s' : State} {t ry r : Nat} (h : ICy p D σ t ry r s) + {ws : List (Ptr × Nat)} (hP : PPostB D s s' ws) (hc : icyChk p ws ry r = true) : ICy p D σ t ry r s' := by + simp only [icyChk, Bool.and_eq_true] at hc + have L := h.l.st.lay + exact ⟨h.l.step hP hc.1.1, Fam.keep L hP hc.1.2 h.y, Fam.keep L hP hc.2 h.yh⟩ + +/-- What `ŷ[r]` needs of the layout, with `ry` polynomials of `y`. -/ +def yhChk (p : Params) (ry r : Nat) : Bool := + let y := pS (yBase p + r) + let yh := pS (yhBase p + r) + copyChk (sgB p) (sgW p) yh y 1024 && icyChk p [(yh, 1024)] ry r && ipChk (sgB p) (sgW p) yh && + icyChk p [(yh, 1024), (sc oPS, 1024)] ry r + +theorem yhR_ok {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} {σ : State} {t ry r : Nat} + (hc : yhChk p ry r = true) (hr : r < ry) {s : State} (h : ICy p D σ t ry r s) : + WP isa (yhR P p r) s (ICy p D σ t ry (r + 1)) := by + simp only [yhChk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨cc, c3⟩, ci⟩, c4⟩ := hc + unfold yhR + refine WP.seq (WP.mono (copy_okB h.l.st.lay cc) fun s3 ⟨hP3, _, hb3⟩ => ?_) + have I3 := h.step hP3 c3 + have hyh3 : Pl s3 (yhBase p + r) (Yv p σ (p.ℓ * t) r) := by + show PolyIs _ _ _ + rw [hP3.pa (pS_bases _)] + exact polyIs_of_bytes hb3 (h.y r hr) + refine WP.mono (ipAt_ok (t := ntt) hP.ntt I3.l.st.lay ci hyh3.1) fun s4 ⟨hP4, _, hq4⟩ => ?_ + have I4 := I3.step hP4 c4 + refine ⟨I4.l, I4.y, Fam.snoc I4.yh ?_⟩ + show PolyIs _ _ _ + rw [hP4.pa (pS_bases _), hyh3.2] at * + exact hq4 + +/-- Iteration `t`, with the first `4g` polynomials of `y` and `ŷ`, and the first `k` seeds of `MS4`. -/ +structure SD (p : Params) (D : Nat) (σ : State) (t g k : Nat) (s : State) : Prop where + m : ICm p D σ t (4 * g) s + sd : ∀ j < k, bytesAt s.mem (pa s (sc (oMS4 + 66 * j))) 66 = + rppOf p σ ++ integerToBytes (p.ℓ * t + (4 * g + j)) 2 + +/-- What seed `k` of `MS4` needs of the layout. -/ +def ms4Chk (p : Params) (g k : Nat) : Bool := + let w1 : List (Ptr × Nat) := [(sc (oMS4 + 66 * k), 64)] + let w2 : List (Ptr × Nat) := [(sc (oMS4 + 66 * k + 64), 2)] + copyChk (sgB p) (sgW p) (sc (oMS4 + 66 * k)) (sc oMS) 64 && icmChk p w1 (4 * g) && icmChk p w2 (4 * g) && + inB (sgB p) (sc oKAP) 8 && inB (sgW p) (sc (oMS4 + 66 * k + 64)) 2 && + keepB (sgB p) w2 (sc (oMS4 + 66 * k)) 64 && + (List.range k).all (fun j => keepB (sgB p) w1 (sc (oMS4 + 66 * j)) 66 && + keepB (sgB p) w2 (sc (oMS4 + 66 * j)) 66) && + decide (p.ℓ * 813 + (4 * g + k) < 2 ^ 16) && decide (4 * g + k < 2 ^ 31) + +theorem ms4_ok {p : Params} {D : Nat} {σ : State} {t g k : Nat} (hc : ms4Chk p g k = true) {s : State} + (h : SD p D σ t g k s) : WP isa (cpM4 g k) s (SD p D σ t g (k + 1)) := by + simp only [ms4Chk, Bool.and_eq_true, decide_eq_true_eq] at hc + obtain ⟨⟨⟨⟨⟨⟨⟨⟨cc, c1⟩, c2⟩, k1⟩, w2⟩, k12⟩, kd⟩, hx⟩, hr⟩ := hc + have ht := h.m.l.t_lt + have hx' : p.ℓ * t + (4 * g + k) < 2 ^ 16 := by + have := Nat.mul_le_mul_left p.ℓ (show t ≤ 813 by omega); omega + unfold cpM4 + refine WP.seq (WP.mono (copy_okB h.m.l.st.lay cc) fun s1 ⟨hP1, _, hb1⟩ => ?_) + have I1 := h.m.step hP1 c1 + refine WP.mono (setKap_okB I1.l.st.lay hr hx' k1 w2 I1.l.kap) fun s2 ⟨hP2, _, hb2⟩ => ?_ + have I2 := I1.step hP2 c2 + refine ⟨I2, fun j hj => ?_⟩ + rcases (by omega : j < k ∨ j = k) with hj' | rfl + · have kk := List.all_eq_true.mp kd j (List.mem_range.mpr hj') + simp only [Bool.and_eq_true] at kk + rw [I1.l.st.lay.keepBytes hP2 kk.2, h.m.l.st.lay.keepBytes hP1 kk.1] + exact h.sd j hj' + · rw [VG.Proof.MlKem.bytesAt_add _ _ 64 2, pa_sc_add, I1.l.st.lay.keepBytes hP2 k12, hP1.pa (sc_bases _), hb1, + h.m.l.k.rpp, hP2.pa (sc_bases _), hb2] + +/-- What `y[4g], …, y[4g + 3]` and their `ŷ` need of the layout. -/ +def m4Chk (p : Params) (g : Nat) : Bool := + (List.range 4).all (ms4Chk p g) && mask4Chk (sgB p) (sgW p) (yP p (4 * g)) (r4P p) && + icmChk p [(yP p (4 * g), 4096), (r4P p, 8192)] (4 * g) && + (List.range 4).all (fun r => yhChk p (4 * g + 4) (4 * g + r)) && decide (p.γ₁ = 2 ^ 17 ∨ p.γ₁ = 2 ^ 19) + +theorem m4Chk_spec {p : Params} {g : Nat} (hc : m4Chk p g = true) : + (∀ k < 4, ms4Chk p g k = true) ∧ mask4Chk (sgB p) (sgW p) (yP p (4 * g)) (r4P p) = true ∧ + icmChk p [(yP p (4 * g), 4096), (r4P p, 8192)] (4 * g) = true ∧ + (∀ r, 4 * g ≤ r → r < 4 * g + 4 → yhChk p (4 * g + 4) r = true) ∧ (p.γ₁ = 2 ^ 17 ∨ p.γ₁ = 2 ^ 19) := by + simp only [m4Chk, Bool.and_eq_true, List.all_eq_true, List.mem_range, decide_eq_true_eq] at hc + obtain ⟨⟨⟨⟨hcp, cm⟩, c2⟩, hyh⟩, hγ⟩ := hc + refine ⟨hcp, cm, c2, fun r h1 h2 => ?_, hγ⟩ + have := hyh (r - 4 * g) (by omega) + rwa [show 4 * g + (r - 4 * g) = r by omega] at this + +/-- The call of `vg_mldsa_expand_mask_poly4`, from the four seeds of `MS4`. -/ +theorem m4call_ok {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} {σ : State} {t g : Nat} + (hγ : p.γ₁ = 2 ^ 17 ∨ p.γ₁ = 2 ^ 19) (cm : mask4Chk (sgB p) (sgW p) (yP p (4 * g)) (r4P p) = true) + (c2 : icmChk p [(yP p (4 * g), 4096), (r4P p, 8192)] (4 * g) = true) {s : State} (h : SD p D σ t g 4 s) : + WP isa (mask4At P p.γ₁ (yP p (4 * g)) (r4P p)) s (ICy p D σ t (4 * g + 4) (4 * g)) := by + refine WP.mono (mask4Call_ok hP h.m.l.st.lay hγ cm) fun s2 ⟨hP2, _, hq2⟩ => ?_ + have I2 := h.m.step hP2 c2 + refine ⟨I2.l, fun j hj => ?_, I2.yh⟩ + rcases (by omega : j < 4 * g ∨ 4 * g ≤ j) with hj' | hj' + · exact I2.y j hj' + · obtain ⟨k, hk, rfl⟩ : ∃ k, k < 4 ∧ j = 4 * g + k := ⟨j - 4 * g, by omega, by omega⟩ + have hq := hq2 k hk + rw [seed66, pa_sc_add, h.sd k hk] at hq + show PolyIs s2.mem (pa s2 (pS (yBase p + (4 * g + k)))) _ + rw [hP2.pa (pS_bases _), ← Nat.add_assoc, ← pa_poly4] + exact hq + +theorem mask4_ok {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} {σ : State} {t g : Nat} + (hc : m4Chk p g = true) {s : State} (h : ICm p D σ t (4 * g) s) : + WP isa (mask4 P p g) s (ICm p D σ t (4 * (g + 1))) := by + obtain ⟨hcp, cm, c2, hyh, hγ⟩ := m4Chk_spec hc + unfold mask4 + refine WP.seq (WP.mono (seqR_ok (I := fun k => SD p D σ t g k) 4 0 + (fun k _ hk s hs => ms4_ok (hcp k (by omega)) hs) s ⟨h, fun _ h => absurd h (Nat.not_lt_zero _)⟩) + fun s1 hs1 => ?_) + rw [Nat.zero_add] at hs1 + refine WP.seq (WP.mono (m4call_ok hP hγ cm c2 hs1) fun s2 hs2 => ?_) + exact WP.mono (seqR_ok (I := fun r => ICy p D σ t (4 * g + 4) r) 4 (4 * g) + (fun r h1 hr s hs => yhR_ok hP (hyh r h1 hr) (by omega) hs) s2 hs2) + fun s3 hs3 => ⟨hs3.l, hs3.y, hs3.yh⟩ + /-! ## `w` -/ /-- Iteration `t`, with `y`, `ŷ`, and the first `i` polynomials of `w`. -/ @@ -357,7 +494,7 @@ structure IC (p : Params) (D : Nat) (σ : State) (t : Nat) (s : State) : Prop wh /-- What the commitment needs of the layout. -/ def cChk (p : Params) : Bool := - (List.range p.ℓ).all (mChk p) && (List.range p.k).all (wChk p) && (List.range p.k).all (hChk p) && + (List.range (p.ℓ / 4)).all (m4Chk p) && (List.range p.ℓ).all (mChk p) && (List.range p.k).all (wChk p) && (List.range p.k).all (hChk p) && shakeChk (sgB p) (sgW p) [((.r12, 0), 64), (sc oW1, p.k * w1Len p)] (sc oCT) (cLen p) && icwChk p [(sc 0, 200), (sc 200, 640), (sc oCT, cLen p)] p.k @@ -367,12 +504,15 @@ theorem cChk_ok {p : Params} (h : Ok3 p) : cChk p = true := by theorem commit_ok {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} (hc : cChk p = true) {σ : State} {t : Nat} {s : State} (h : IL p D σ t s) : WP isa (commit P p) s (IC p D σ t) := by simp only [cChk, Bool.and_eq_true, List.all_eq_true, List.mem_range] at hc - obtain ⟨⟨⟨⟨hm, hw⟩, hh⟩, hs⟩, hk⟩ := hc + obtain ⟨⟨⟨⟨⟨hm4, hm⟩, hw⟩, hh⟩, hs⟩, hk⟩ := hc unfold commit - refine WP.seq (WP.mono (seqR_ok (I := fun r => ICm p D σ t r) p.ℓ 0 - (fun r _ hr s hs => maskR_ok hP (hm r (by omega)) hs) s - ⟨h, fun _ h => absurd h (Nat.not_lt_zero _), fun _ h => absurd h (Nat.not_lt_zero _)⟩) fun s1 hs1 => ?_) - rw [Nat.zero_add] at hs1 + refine WP.seq (WP.mono (seqR_ok (I := fun g => ICm p D σ t (4 * g)) (p.ℓ / 4) 0 + (fun g _ hg s hs => mask4_ok hP (hm4 g (by omega)) hs) s + ⟨h, fun _ h => absurd h (Nat.not_lt_zero _), fun _ h => absurd h (Nat.not_lt_zero _)⟩) fun s0 hs0 => ?_) + rw [Nat.zero_add] at hs0 + refine WP.seq (WP.mono (seqR_ok (I := fun r => ICm p D σ t r) (p.ℓ % 4) (4 * (p.ℓ / 4)) + (fun r _ hr s hs => maskR_ok hP (hm r (by omega)) hs) s0 hs0) fun s1 hs1 => ?_) + rw [Nat.div_add_mod] at hs1 refine WP.seq (WP.mono (seqR_ok (I := fun i => ICw p D σ t i) p.k 0 (fun i _ hi s hs => rowW_ok hP (hw i (by omega)) (by omega) hs) s1 ⟨hs1.l, hs1.y, hs1.yh, fun _ h => absurd h (Nat.not_lt_zero _)⟩) fun s2 hs2 => ?_) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseCCT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseCCT.lean index f87dc034b..f157e8f8e 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseCCT.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseCCT.lean @@ -55,7 +55,7 @@ theorem setKappa_post {D : Nat} {rbs wbs : List (Reg × Nat)} {s : State} (L : L rwa [BitVec.add_zero] at this have i1 : InRegions s.wr (pa s (sc (oMS + 65))) 1 := by rw [e65]; exact inRegions_sub (off := 1) (l := 1) w2 (by omega) (by decide) - refine WP.mono (setKappa_ok r hr s (L.iR h1) i0 i1) fun s' ⟨hm, k⟩ => ⟨_, (postB_of_keep (D := D) k (by decide) + refine WP.mono (setKap_ok (oMS + 64) r hr s (L.iR h1) i0 i1) fun s' ⟨hm, k⟩ => ⟨_, (postB_of_keep (D := D) k (by decide) (W := [⟨pa s (sc (oMS + 64)), 2⟩]) ?_).1⟩ rw [hm] exact ((Frame.refl _ _).writeW (List.mem_singleton_self _) _ c0).writeW (List.mem_singleton_self _) _ c1 @@ -81,6 +81,56 @@ theorem maskR_trL {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} {r : Nat (ipAt_tr (t := ntt) hP.ntt ci)))) (fun _ _ h => ⟨h, trivial, trivial⟩) fun _ _ h => h +/-! ## `y` and `ŷ`, four at a time -/ + +theorem ms4_trL {D : Nat} {p : Params} {g k : Nat} (hc : ms4Chk p g k = true) : + RelCT isa (LRel D (sgR p) (sgW p)) (cpM4 g k) fun _ _ => True := by + simp only [ms4Chk, Bool.and_eq_true] at hc + have cc := hc.1.1.1.1.1.1.1.1 + have hcc := copyChk_spec cc + unfold cpM4 + exact RelCT.mono (seqL (p := p) (I := fun _ => True) (J := fun _ => True) + (RelCT.mono (copy_tr hcc.2.2.2.1 hcc.2.2.2.2.1 hcc.2.2.2.2.2.1 hcc.2.2.2.2.2.2.1 hcc.2.2.2.2.2.2.2 + fun x y h => ⟨lrel_rbx h.1 _ (List.mem_singleton_self _), lrel_rbx h.1 _ (List.mem_singleton_self _)⟩) + (fun _ _ h => h) fun _ _ h => h) + (fun x L _ => WP.mono (copy_okB L cc) fun _ ⟨hP', _⟩ => ⟨⟨_, hP'⟩, trivial⟩) + (block_tr rfl fun x y h => lrel_rbx h.1)) + (fun _ _ h => ⟨h, trivial, trivial⟩) fun _ _ h => h + +theorem yhR_trL {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} {t ry r : Nat} (hc : yhChk p ry r = true) + (hr : r < ry) : + RelCT isa (fun x y => LRel D (sgR p) (sgW p) x y ∧ (∃ σ, ICy p D σ t ry r x) ∧ ∃ σ, ICy p D σ t ry r y) + (yhR P p r) fun _ _ => True := by + simp only [yhChk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨cc, _⟩, ci⟩, _⟩ := hc + have hcc := copyChk_spec cc + unfold yhR + exact seqL (J := fun s => Reduced s.mem (pa s (yhP p r))) + (RelCT.mono (copy_tr hcc.2.2.2.1 hcc.2.2.2.2.1 hcc.2.2.2.2.2.1 hcc.2.2.2.2.2.2.1 hcc.2.2.2.2.2.2.2 + fun x y h => ⟨lrel_rbx h.1 _ (List.mem_singleton_self _), lrel_rbx h.1 _ (List.mem_singleton_self _)⟩) + (fun _ _ h => h) fun _ _ h => h) + (fun x L ⟨_, I⟩ => WP.mono (copy_okB L cc) fun x' ⟨hP', _, hb⟩ => + ⟨⟨_, hP'⟩, by rw [hP'.pa (pS_bases _)]; exact reduced_congr₂ (bytes_of_bytesAt hb) (I.y r hr).1⟩) + (ipAt_tr (t := ntt) hP.ntt ci) + +theorem mask4_tr {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} {g : Nat} (hc : m4Chk p g = true) + {E : State → State → Prop} {t : Nat} : + RelCT isa (RS p D E fun σ s => ICm p D σ t (4 * g) s) (mask4 P p g) + (RS p D E fun σ s => ICm p D σ t (4 * (g + 1)) s) := by + obtain ⟨hcp, cm, c2, hyh, hγ⟩ := m4Chk_spec hc + unfold mask4 + refine RelCT.seq (R := RS p D E fun σ s => SD p D σ t g 4 s) ?_ + (RelCT.seq (R := RS p D E fun σ s => ICy p D σ t (4 * g + 4) (4 * g) s) ?_ ?_) + · refine RelCT.mono (seqR_tr (R := fun k => RS p D E fun σ s => SD p D σ t g k s) 4 0 fun k _ hk => + liftT (fun _ _ h => h.m.l.st) (fun _ _ _ h => ms4_ok (hcp k (by omega)) h) (ms4_trL (hcp k (by omega)))) + (fun x y h => h.mono (fun _ _ h => h) fun _ _ h => ⟨h, fun _ h => absurd h (Nat.not_lt_zero _)⟩) + fun x y h => by rwa [Nat.zero_add] at h + · exact liftT (fun _ _ h => h.m.l.st) (fun _ _ _ h => m4call_ok hP hγ cm c2 h) (mask4Call_tr hP hγ cm) + · exact RelCT.mono (seqR_tr (R := fun r => RS p D E fun σ s => ICy p D σ t (4 * g + 4) r s) 4 (4 * g) + fun r h1 hr => liftL (T := fun s => ∃ σ, ICy p D σ t (4 * g + 4) r s) (fun σ s h => ⟨h.l.st, σ, h⟩) + (fun _ _ _ h => yhR_ok hP (hyh r h1 hr) (by omega) h) (yhR_trL hP (hyh r h1 hr) (by omega))) + (fun x y h => h) fun x y h => h.mono (fun _ _ h => h) fun _ _ h => ⟨h.l, h.y, h.yh⟩ + /-! ## `w` -/ /-- `w[i]` from runs in iteration `t` with `y`, `ŷ` and the first `i` polynomials of `w`. -/ @@ -158,15 +208,21 @@ theorem commit_tr {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} (hc : cC RelCT isa (RS p D E fun σ s => IL p D σ t s) (commit P p) (RS p D E fun σ s => IC p D σ t s) := by have hc' := hc simp only [cChk, Bool.and_eq_true, List.all_eq_true, List.mem_range] at hc' - obtain ⟨⟨⟨⟨hm, hw⟩, hh⟩, hs⟩, _⟩ := hc' + obtain ⟨⟨⟨⟨⟨hm4, hm⟩, hw⟩, hh⟩, hs⟩, _⟩ := hc' unfold commit - refine RelCT.seq (R := RS p D E fun σ s => ICw p D σ t 0 s) ?_ (RelCT.seq (R := RS p D E fun σ s => ICh p D σ t 0 s) - ?_ (RelCT.seq (R := RS p D E fun σ s => ICh p D σ t p.k s) ?_ ?_)) - · refine RelCT.mono (seqR_tr (R := fun r => RS p D E fun σ s => ICm p D σ t r s) p.ℓ 0 fun r _ hr => - liftT (fun _ _ h => h.l.st) (fun _ _ _ h => maskR_ok hP (hm r (by omega)) h) (maskR_trL hP (hm r (by omega)))) + refine RelCT.seq (R := RS p D E fun σ s => ICm p D σ t (4 * (p.ℓ / 4)) s) ?_ + (RelCT.seq (R := RS p D E fun σ s => ICw p D σ t 0 s) ?_ (RelCT.seq (R := RS p D E fun σ s => ICh p D σ t 0 s) + ?_ (RelCT.seq (R := RS p D E fun σ s => ICh p D σ t p.k s) ?_ ?_))) + · refine RelCT.mono (seqR_tr (R := fun g => RS p D E fun σ s => ICm p D σ t (4 * g) s) (p.ℓ / 4) 0 fun g _ hg => + mask4_tr hP (hm4 g (by omega))) (fun x y h => h.mono (fun _ _ h => h) fun _ _ h => ⟨h, fun _ h => absurd h (Nat.not_lt_zero _), fun _ h => absurd h (Nat.not_lt_zero _)⟩) - fun x y h => by rw [Nat.zero_add] at h; exact h.mono (fun _ _ h => h) fun _ _ h => + fun x y h => by rwa [Nat.zero_add] at h + · refine RelCT.mono (seqR_tr (R := fun r => RS p D E fun σ s => ICm p D σ t r s) (p.ℓ % 4) (4 * (p.ℓ / 4)) + fun r _ hr => + liftT (fun _ _ h => h.l.st) (fun _ _ _ h => maskR_ok hP (hm r (by omega)) h) (maskR_trL hP (hm r (by omega)))) + (fun x y h => h) + fun x y h => by rw [Nat.div_add_mod] at h; exact h.mono (fun _ _ h => h) fun _ _ h => ⟨h.l, h.y, h.yh, fun _ h => absurd h (Nat.not_lt_zero _)⟩ · refine RelCT.mono (seqR_tr (R := fun i => RS p D E fun σ s => ICw p D σ t i s) p.k 0 fun i _ hi => liftL (T := fun s => ∃ σ, ICw p D σ t i s) (fun σ s h => ⟨h.l.st, σ, h⟩) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Prims.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Prims.lean index dee6add18..f95313642 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Prims.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Prims.lean @@ -44,6 +44,7 @@ structure PrimsOk (P : Prims) (D : Nat) where bitUnpack : Callee (fun S => bitUnpackContract X86_64.abi S) D P.bitUnpack hintBitPack : Callee (fun S => hintBitPackContract X86_64.abi S) D P.hintBitPack rej4 : Callee (fun S => rejNTT4Contract X86_64.abi S) D P.rej4 + expandMask4 : Callee (fun S => expandMask4Contract X86_64.abi S) D P.expandMask4 /-- `vg_mldsa_rej_ntt_poly`'s result depends only on its public data (its seed). -/ rejRet : RetPub (rejNTTContract X86_64.abi rejNTT.S) P.rejNTT /-- `vg_mldsa_rej_ntt_poly` succeeds only if `RejNTTPoly` finishes within `maxBounds`. -/ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PrimsB.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PrimsB.lean index 780c39403..66afdca19 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PrimsB.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PrimsB.lean @@ -385,6 +385,94 @@ theorem maskAt_tr {P : Prims} (hP : PrimsOk P D) {γ : Nat} {a : Ptr} (hγ : γ simp only [hx1, hx2, hx3, hx4, hy1, hy2, hy3, hy4, Arg.val, R.pa i1, R.pa i2, R.pa i3, (At.of R.lx hmx kx).rsp, (At.of R.ly hmy ky).rsp, R.rsp, and_self] +/-! ## `ExpandMask` four times -/ + +/-- What the call of `vg_mldsa_expand_mask_poly4` from the seeds at `MS4` to the four polynomials from `a`, +with the working space `w`, needs of the layout. -/ +def mask4Chk (bs wbs : List (Reg × Nat)) (a w : Ptr) : Bool := + inB wbs a 4096 && inB wbs w 8192 && inB bs (sc oMS4) 264 && inB bs a 4096 && inB bs w 8192 && + sepB bs (sc oMS4) 264 a 4096 && sepB bs (sc oMS4) 264 w 8192 && sepB bs a 4096 w 8192 && + decide (a.1 ∈ bases) && decide (a.2 < 2 ^ 31) && decide (w.1 ∈ bases) && decide (w.2 < 2 ^ 31) + +theorem mask4Chk_spec {bs wbs : List (Reg × Nat)} {a w : Ptr} (hc : mask4Chk bs wbs a w = true) : + inB wbs a 4096 = true ∧ inB wbs w 8192 = true ∧ inB bs (sc oMS4) 264 = true ∧ inB bs a 4096 = true ∧ + inB bs w 8192 = true ∧ sepB bs (sc oMS4) 264 a 4096 = true ∧ sepB bs (sc oMS4) 264 w 8192 = true ∧ + sepB bs a 4096 w 8192 = true ∧ a.1 ∈ bases ∧ a.2 < 2 ^ 31 ∧ w.1 ∈ bases ∧ w.2 < 2 ^ 31 := by + simp only [mask4Chk, Bool.and_eq_true, decide_eq_true_eq] at hc + obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨h1, h2⟩, h3⟩, h4⟩, h5⟩, h6⟩, h7⟩, h8⟩, h9⟩, h10⟩, h11⟩, h12⟩ := hc + exact ⟨h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11, h12⟩ + +theorem mask4Pre {S : Nat} (hS : S + 8 ≤ D) {s s1 : State} (A : At D rbs wbs s s1) {γ : Nat} {a w : Ptr} + (hγ : γ = 2 ^ 17 ∨ γ = 2 ^ 19) (hc : mask4Chk (rbs ++ wbs) wbs a w = true) + (hA : ArgsIn [.ptr (sc oMS4), .imm γ, .ptr a, .ptr w] s s1) : + (expandMask4Contract X86_64.abi S).pre + (s1.callEntry.withRegions [⟨pa s (sc oMS4), 264⟩] [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩]) := by + obtain ⟨_, _, i1, i2, i3, d12, d13, d23, _⟩ := mask4Chk_spec hc + obtain ⟨e1, e2, e3, e4⟩ := argsIn4 hA + have hD : 8 ≤ D := by omega + have hγ' : γ < 2 ^ 32 := by omega + have hwf := ce_wfS (ws := [64, 32, 64, 64]) (by decide) hS (A.rsp ▸ A.L.sp) [⟨pa s (sc oMS4), 264⟩] + [⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩] + sig_pre [expandMask4Contract, expandMask4Sig, X86_64.abi, VG.X86_64.argRegs] + simp only [e1, e2, e3, e4, Arg.val, sw32_ofNat hγ'] + refine ⟨hwf, trivial, trivial, A.L.disj d12, A.L.disj d13, A.L.disj d23, A.ret i1 hD, A.ret i2 hD, ?_, + A.L.nwp i1, A.L.nwp i2, A.L.nwp i3, hγ⟩ + refine Sig.conj_cons.mpr ⟨A.ret i3 hD, conj_stk [⟨pa s (sc oMS4), 264⟩, ⟨pa s a, 4096⟩, ⟨pa s w, 8192⟩] ?_⟩ + simp only [List.mem_cons, List.mem_nil_iff, or_false, forall_eq_or_imp, forall_eq] + exact ⟨A.stk hS i1, A.stk hS i2, A.stk hS i3⟩ + +/-- The seeds on entry to the call are those before it. -/ +theorem At.seeds66 {s s1 : State} (A : At D rbs wbs s s1) (hi : inB (rbs ++ wbs) (sc oMS4) 264 = true) (hD : 8 ≤ D) + {k : Nat} (hk : k < 4) : + seed66 (s1.mem.writeW (s1.gpr .rsp - 8) (s1.unknowns 0)) (pa s (sc oMS4)) k = seed66 s.mem (pa s (sc oMS4)) k := by + unfold seed66 + rw [← VG.Proof.MlKem.bytesAt_slice _ _ (show 66 * k + 66 ≤ 264 by omega), + ← VG.Proof.MlKem.bytesAt_slice s.mem _ (show 66 * k + 66 ≤ 264 by omega), A.bytes' hi hD] + +/-- The call of `vg_mldsa_expand_mask_poly4`. -/ +theorem mask4Call_ok {P : Prims} (hP : PrimsOk P D) {s : State} (L : Lay D rbs wbs s) {γ : Nat} {a w : Ptr} + (hγ : γ = 2 ^ 17 ∨ γ = 2 ^ 19) (hc : mask4Chk (rbs ++ wbs) wbs a w = true) : + WP isa (mask4At P γ a w) s fun s' => PPostB D s s' [(a, 4096), (w, 8192)] ∧ + (∀ r ∈ calleeSaved, s'.gpr r = s.gpr r) ∧ ∀ k < 4, + PolyIs s'.mem (poly4 (pa s a) k) (toRq (bitUnpack + (H (seed66 s.mem (pa s (sc oMS4)) k) (32 * (1 + bitlen (γ - 1)))) (γ - 1) γ)) := by + obtain ⟨w1, w2, i1, i2, i3, _, _, _, b1, o1, b2, o2⟩ := mask4Chk_spec hc + have hD : 8 ≤ D := by have := hP.expandMask4.hS; omega + have hγ' : γ < 2 ^ 32 := by omega + refine WP.mono (callP_ok hP.expandMask4.ver.1 hP.expandMask4.nosp hP.expandMask4.depth L.dsm + (by simp only [List.all_cons, List.all_nil, Arg.ok, b1, o1, b2, o2, decide_true, Bool.and_true, + decide_eq_true hγ']; decide) + (fun s1 hA hm k => mask4Pre hP.expandMask4.hS (At.of L hm k) hγ hc hA) + (covers_append (L.cR i1) (covers_wr (covers_cons (L.cW w1) (L.cW w2)))) (covers_cons (L.cW w1) (L.cW w2))) + fun s' ⟨hpost, hcs, s1, hA, hm, k, s₂, hm₂, _, hq⟩ => ⟨hpost, hcs, ?_⟩ + have A := At.of L hm k + obtain ⟨e1, e2, e3, _⟩ := argsIn4 hA + sig_post [expandMask4Contract, expandMask4Sig, X86_64.abi, VG.X86_64.argRegs] at hq + simp only [e1, e2, e3, Arg.val, hm₂, sw32_ofNat hγ'] at hq + exact fun k hk => by rw [← A.seeds66 i1 hD hk]; exact hq k hk + +theorem mask4Call_tr {P : Prims} (hP : PrimsOk P D) {γ : Nat} {a w : Ptr} (hγ : γ = 2 ^ 17 ∨ γ = 2 ^ 19) + (hc : mask4Chk (rbs ++ wbs) wbs a w = true) : + RelCT isa (LRel D rbs wbs) (mask4At P γ a w) fun _ _ => True := by + obtain ⟨w1, w2, i1, i2, i3, _, _, _, b1, o1, b2, o2⟩ := mask4Chk_spec hc + have hγ' : γ < 2 ^ 32 := by omega + refine callP_tr hP.expandMask4.ver.1 hP.expandMask4.ver.2.1 + (by simp only [List.all_cons, List.all_nil, Arg.ok, b1, o1, b2, o2, decide_true, Bool.and_true, + decide_eq_true hγ']; decide) + fun x y x1 y1 R ⟨⟨hAx, hmx⟩, kx⟩ ⟨⟨hAy, hmy⟩, ky⟩ => + ⟨_, _, _, _, mask4Pre hP.expandMask4.hS (At.of R.lx hmx kx) hγ hc hAx, + mask4Pre hP.expandMask4.hS (At.of R.ly hmy ky) hγ hc hAy, ?_, + by rw [kx.2.1, kx.2.2]; exact covers_append (R.lx.cR i1) (covers_wr (covers_cons (R.lx.cW w1) (R.lx.cW w2))), + by rw [kx.2.2]; exact covers_cons (R.lx.cW w1) (R.lx.cW w2), + by rw [ky.2.1, ky.2.2]; exact covers_append (R.ly.cR i1) (covers_wr (covers_cons (R.ly.cW w1) (R.ly.cW w2))), + by rw [ky.2.2]; exact covers_cons (R.ly.cW w1) (R.ly.cW w2), + by rw [(At.of R.lx hmx kx).rsp, (At.of R.ly hmy ky).rsp, R.rsp]⟩ + obtain ⟨hx1, hx2, hx3, hx4⟩ := argsIn4 hAx + obtain ⟨hy1, hy2, hy3, hy4⟩ := argsIn4 hAy + sig_pub [expandMask4Contract, expandMask4Sig, X86_64.abi, VG.X86_64.argRegs] + simp only [hx1, hx2, hx3, hx4, hy1, hy2, hy3, hy4, Arg.val, R.pa i1, R.pa i2, R.pa i3, + (At.of R.lx hmx kx).rsp, (At.of R.ly hmy ky).rsp, R.rsp, and_self] + /-! ## `SampleInBall` -/ /-- What a call of `vg_mldsa_sample_in_ball` of the `len` bytes at `CT` to `c` needs of the layout. -/ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean index 62ea144d6..a6d83b621 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean @@ -75,8 +75,8 @@ with no implementation of it. -/ theorem sign_same {m mc : Prog isa → Bool} (hm : Comp m mc) {B : Backend} (h1 : mc B.ntt = true) (h2 : mc B.invNtt = true) (h3 : mc B.mul = true) (h4 : mc B.mulAdd = true) (h5 : mc B.add = true) - (h6 : mc B.sub = true) (h7 : mc B.rej4 = true) (h8 : mc B.highBits = true) (h9 : mc B.lowBits = true) - (h10 : mc B.normLt = true) (h11 : mc B.makeHint = true) (p : Params) : + (h6 : mc B.sub = true) (h7 : mc B.rej4 = true) (h8 : mc B.expandMask4 = true) (h9 : mc B.highBits = true) + (h10 : mc B.lowBits = true) (h11 : mc B.normLt = true) (h12 : mc B.makeHint = true) (p : Params) : Same m (Impl.MlDsa.X86_64.Sign.sign (primsWith B) p) (Impl.MlDsa.X86_64.Sign.sign (primsWith .empty) p) := by unfold Impl.MlDsa.X86_64.Sign.sign same_tac hm @@ -93,14 +93,15 @@ include h3 theorem sign_ctl : ctlOk (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p) = true := ctlOk_of_ctlC (Same.ok (sign_same Comp.ctlC v.ok.ntt.ctl v.ok.invNtt.ctl v.ok.mul.ctl v.ok.mulAdd.ctl - v.ok.add.ctl v.ok.sub.ctl v.ok.rej4.ctl v.ok.highBits.ctl v.ok.lowBits.ctl v.ok.normLt.ctl v.ok.makeHint.ctl p) - (sign0_ctlC h3)) + v.ok.add.ctl v.ok.sub.ctl v.ok.rej4.ctl v.ok.expandMask4.ctl v.ok.highBits.ctl v.ok.lowBits.ctl + v.ok.normLt.ctl v.ok.makeHint.ctl p) (sign0_ctlC h3)) theorem sign_spSafe : (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p).all (fun i => !isa.writesSp i) = true := Code.all_of_allInstrs (Same.ok (sign_same (Comp.all _) (Code.allInstrs_of_all v.ok.ntt.sp) (Code.allInstrs_of_all v.ok.invNtt.sp) (Code.allInstrs_of_all v.ok.mul.sp) (Code.allInstrs_of_all v.ok.mulAdd.sp) (Code.allInstrs_of_all v.ok.add.sp) (Code.allInstrs_of_all v.ok.sub.sp) (Code.allInstrs_of_all v.ok.rej4.sp) + (Code.allInstrs_of_all v.ok.expandMask4.sp) (Code.allInstrs_of_all v.ok.highBits.sp) (Code.allInstrs_of_all v.ok.lowBits.sp) (Code.allInstrs_of_all v.ok.normLt.sp) (Code.allInstrs_of_all v.ok.makeHint.sp) p) (sign0_sp h3)) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTCompute.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTCompute.lean index 7a0d38ea9..4819fc76d 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTCompute.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTCompute.lean @@ -104,7 +104,7 @@ theorem dot_tr : RelCT isa (RV p (IR p r)) (dot P p r) · have L := IRX.lrel hp (fun _ _ _ _ _ _ hd => ⟨_, hd.1⟩) hxy have hz : k ≠ 100 := by have := kl_le p hp; omega have red : ∀ {σ s}, VPre p σ → IRX p r (fun _ A' _ σ s₀ s => DI p σ A' r k s₀ s) σ s → - Reduced s.mem (pa s pW) ∧ Reduced s.mem (pa s (pA r k)) ∧ Reduced s.mem (pa s (pZ k)) := + Reduced s.mem (pa s pW) ∧ Reduced s.mem (pa s (pA p.ℓ r k)) ∧ Reduced s.mem (pa s (pZ k)) := fun hv ⟨_, _, _, _, s₀, hs, hP, _, hW⟩ => by have L₀ := hs.t.lay hp hv have H := hP.mono (sub1 (wsR_mem p r).1) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean index c185e4bad..ca3653a6a 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/CTSample.lean @@ -3,9 +3,9 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Verify.CTStages /-! # ML-DSA verification on x86-64: constant time, the samplers -The seed of each entry of `Â` is `ρ ‖ s ‖ r`, and `c̃` a piece of the -signature, public in both runs; each sampler's output is masked with its -result without a branch (`sampledTail_tr`). +The seed of each entry of `Â` (and of each four, `aGrp_tr`) is `ρ ‖ s ‖ r`, +and `c̃` a piece of the signature, public in both runs; each sampler's output +is masked with its result without a branch (`sampledTail_tr`). -/ namespace VG.Proof.MlDsa.X86_64.Verify @@ -23,33 +23,31 @@ def HN (p : Params) (σ : State) (h : List (Vector Bool n)) : Prop := /-- Before the samplers, after them and while sampling `Â`. -/ def Is (p : Params) (σ s : State) : Prop := ∃ h, HN p σ h ∧ S2 p h p.ℓ σ s ∧ s.gpr .r15 = flag True def I4 (p : Params) (σ s : State) : Prop := ∃ h, HN p σ h ∧ S4 p h σ s -def IA (p : Params) (r c : Nat) (σ s : State) : Prop := ∃ h, HN p σ h ∧ S3 p h r c σ s +def IA (p : Params) (e : Nat) (σ s : State) : Prop := ∃ h, HN p σ h ∧ S3 p h e σ s -theorem IA.t {p : Params} {r c : Nat} {σ s : State} (h : IA p r c σ s) : T p σ s := let ⟨_, _, hs⟩ := h; hs.t +theorem IA.t {p : Params} {e : Nat} {σ s : State} (h : IA p e σ s) : T p σ s := let ⟨_, _, hs⟩ := h; hs.t -theorem seed_of {p : Params} {h : List (Vector Bool n)} {r c : Nat} {σ s₀ s : State} (hs : S3 p h r c σ s₀) +theorem seed_of {p : Params} {h : List (Vector Bool n)} {e r c : Nat} {σ s₀ s : State} (hs : S3 p h e σ s₀) (hP : PPostB s₀ s wsB) (hm : s.mem = (s₀.mem.writeW (pa s₀ (sc oSB) + BitVec.ofNat 64 32) (BitVec.ofNat 8 c)).writeW (pa s₀ (sc oSB) + BitVec.ofNat 64 33) (BitVec.ofNat 8 r)) : bytesAt s.mem (pa s (sc oSB)) 34 = aSeed (vPk p σ) r c := by rw [hP.pa (by decide), hm, seed_bytes, hs.rho, aSeed, integerToBytes_one, integerToBytes_one] -theorem aOne_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r c : Nat} (hr : r < p.k) - (hc : c < p.ℓ) : RelCT isa (RV p (IA p r c)) (aOne P (8 * r + c)) (RV p (IA p r (c + 1))) := by - have hck := aChk_all p hp r hr c hc +theorem aOne_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {e : Nat} (he : e < p.k * p.ℓ) : + RelCT isa (RV p (IA p e)) (aOne P p e) (RV p (IA p (e + 1))) := by + have hck := aChk_all p hp e he have hkl := kl_le p hp + obtain ⟨hr, hc⟩ := entry_lt hp he simp only [aChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, Bool.or_eq_true, Bool.not_eq_true', decide_eq_false_iff_not, Nat.not_lt] at hck obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hck - have hT : ∀ σ s, IA p r c σ s → T p σ s := fun _ _ h => h.t - refine relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (aOne_ok C hp hv hr hc hs) fun _ h' => ⟨h, hh, h'⟩) ?_ - have em : (8 * r + c) % 8 = c := by omega - have ed : (8 * r + c) / 8 = r := by omega + have hT : ∀ σ s, IA p e σ s → T p σ s := fun _ _ h => h.t + refine relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (aOne_ok C hp hv he hs) fun _ h' => ⟨h, hh, h'⟩) ?_ unfold aOne - rw [em, ed] - refine RelCT.seq (relInv (I' := fun σ s => ∃ s₀, IA p r c σ s₀ ∧ (PPostB s₀ s wsB ∧ s.gpr .r15 = s₀.gpr .r15 ∧ - s.mem = (s₀.mem.writeW (pa s₀ (sc oSB) + BitVec.ofNat 64 32) (BitVec.ofNat 8 c)).writeW - (pa s₀ (sc oSB) + BitVec.ofNat 64 33) (BitVec.ofNat 8 r))) - (fun σ s hv hs => WP.mono (setB2_ok ((hT σ s hs).lay hp hv) h32 h33 c r (by omega) (by omega)) + refine RelCT.seq (relInv (I' := fun σ s => ∃ s₀, IA p e σ s₀ ∧ (PPostB s₀ s wsB ∧ s.gpr .r15 = s₀.gpr .r15 ∧ + s.mem = (s₀.mem.writeW (pa s₀ (sc oSB) + BitVec.ofNat 64 32) (BitVec.ofNat 8 (e % p.ℓ))).writeW + (pa s₀ (sc oSB) + BitVec.ofNat 64 33) (BitVec.ofNat 8 (e / p.ℓ)))) + (fun σ s hv hs => WP.mono (setB2_ok ((hT σ s hs).lay hp hv) h32 h33 (e % p.ℓ) (e / p.ℓ) (by omega) (by omega)) fun _ h' => ⟨s, hs, h'⟩) (setB2_tr fun x y h => (RV.lrel hp hT h).2.2.1 .rbx (by decide))) ?_ unfold sampled @@ -69,51 +67,51 @@ theorem bytes136 (m : Mem) (P : Addr) : bytesAt m P 136 = bytesAt m P 34 ++ byte show 68 = 34 + 34 from rfl, Proof.MlKem.bytesAt_add] simp only [BitVec.add_assoc, ← BitVec.ofNat_add, List.append_assoc, Nat.reduceAdd] -/-- The four seeds of `SB4`, for the entries `(r, c₀), …, (r, c₀ + 3)`. -/ -theorem GS.seeds {p : Params} {h : List (Vector Bool n)} {r c c₀ : Nat} {σ s : State} (g : GS p h r c c₀ 4 σ s) : - bytesAt s.mem (pa s (sc oSB4)) 136 = aSeed (vPk p σ) r (c₀ + 0) ++ aSeed (vPk p σ) r (c₀ + 1) ++ - aSeed (vPk p σ) r (c₀ + 2) ++ aSeed (vPk p σ) r (c₀ + 3) := by - have e : ∀ k, pa s (sc (oSB4 + 34 * k)) = pa s (sc oSB4) + BitVec.ofNat 64 (34 * k) := fun k => by +/-- The four seeds of `SB4`, for the entries `e, …, e + 3`. -/ +theorem GS.seeds {p : Params} {h : List (Vector Bool n)} {e : Nat} {σ s : State} (g : GS p h e 4 σ s) : + bytesAt s.mem (pa s (sc oSB4)) 136 = aSeed (vPk p σ) ((e + 0) / p.ℓ) ((e + 0) % p.ℓ) ++ + aSeed (vPk p σ) ((e + 1) / p.ℓ) ((e + 1) % p.ℓ) ++ aSeed (vPk p σ) ((e + 2) / p.ℓ) ((e + 2) % p.ℓ) ++ + aSeed (vPk p σ) ((e + 3) / p.ℓ) ((e + 3) % p.ℓ) := by + have ek : ∀ k, pa s (sc (oSB4 + 34 * k)) = pa s (sc oSB4) + BitVec.ofNat 64 (34 * k) := fun k => by simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add] - have b0 : bytesAt s.mem (pa s (sc oSB4)) 34 = aSeed (vPk p σ) r (c₀ + 0) := g.done 0 (by decide) - have b1 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 34) 34 = aSeed (vPk p σ) r (c₀ + 1) := by - have := g.done 1 (by decide); rw [e] at this; exact this - have b2 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 68) 34 = aSeed (vPk p σ) r (c₀ + 2) := by - have := g.done 2 (by decide); rw [e] at this; exact this - have b3 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 102) 34 = aSeed (vPk p σ) r (c₀ + 3) := by - have := g.done 3 (by decide); rw [e] at this; exact this + have b0 : bytesAt s.mem (pa s (sc oSB4)) 34 = aSeed (vPk p σ) ((e + 0) / p.ℓ) ((e + 0) % p.ℓ) := g.done 0 (by decide) + have b1 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 34) 34 = aSeed (vPk p σ) ((e + 1) / p.ℓ) ((e + 1) % p.ℓ) := by + have := g.done 1 (by decide); rw [ek] at this; exact this + have b2 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 68) 34 = aSeed (vPk p σ) ((e + 2) / p.ℓ) ((e + 2) % p.ℓ) := by + have := g.done 2 (by decide); rw [ek] at this; exact this + have b3 : bytesAt s.mem (pa s (sc oSB4) + BitVec.ofNat 64 102) 34 = aSeed (vPk p σ) ((e + 3) / p.ℓ) ((e + 3) % p.ℓ) := by + have := g.done 3 (by decide); rw [ek] at this; exact this rw [bytes136, b0, b1, b2, b3] -theorem slotBlock_tr {r c₀ j : Nat} {P : State → State → Prop} (hP : ∀ s₁ s₂, P s₁ s₂ → s₁.gpr .rbx = s₂.gpr .rbx) : - RelCT isa P (.block (setSR r c₀ j)) fun _ _ => True := +theorem slotBlock_tr {p : Params} {e j : Nat} {P : State → State → Prop} + (hP : ∀ s₁ s₂, P s₁ s₂ → s₁.gpr .rbx = s₂.gpr .rbx) : + RelCT isa P (.block (setSR p e j)) fun _ _ => True := taintRel [.rbx] (fun s₁ s₂ h r hr => by simp only [List.mem_singleton] at hr; subst hr; exact hP s₁ s₂ h) (hc := .block []) (by with_unfolding_all rfl) /-- The bytes of seed `j` of `SB4`, from `GS j` to `GS (j + 1)`. -/ -theorem slot_tr {p : Params} (hp : p ∈ params) {r c c₀ j : Nat} (hr : r < 256) (hj : j < 4) (hx : c₀ + j < 256) - (hck : slotChk p r c j = true) : - RelCT isa (RV p fun σ s => ∃ h, HN p σ h ∧ GS p h r c c₀ j σ s) (.block (setSR r c₀ j)) - (RV p fun σ s => ∃ h, HN p σ h ∧ GS p h r c c₀ (j + 1) σ s) := - relInv (fun σ s hv ⟨h, hh, g⟩ => WP.mono (slot_ok hp hv hr hj hx hck g) fun _ g' => ⟨h, hh, g'⟩) +theorem slot_tr {p : Params} (hp : p ∈ params) {e j : Nat} (hj : j < 4) (he : e + j < p.k * p.ℓ) + (hck : slotChk p e j = true) : + RelCT isa (RV p fun σ s => ∃ h, HN p σ h ∧ GS p h e j σ s) (.block (setSR p e j)) + (RV p fun σ s => ∃ h, HN p σ h ∧ GS p h e (j + 1) σ s) := + relInv (fun σ s hv ⟨h, hh, g⟩ => WP.mono (slot_ok hp hv hj he hck g) fun _ g' => ⟨h, hh, g'⟩) (slotBlock_tr fun x y h => (RV.lrel hp (fun _ _ h => let ⟨_, _, g⟩ := h; g.s3.t) h).2.2.1 .rbx (by decide)) -theorem aGrp_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r c₀ c : Nat} (hr : r < p.k) - (hck : gChk p r c₀ c = true) : RelCT isa (RV p (IA p r c)) (aGrp P p r c₀) (RV p (IA p r (c₀ + 4))) := by - have hkl := kl_le p hp +theorem aGrp_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {g : Nat} + (hck : gChk p (4 * g) = true) : RelCT isa (RV p (IA p (4 * g))) (aGrp P p g) (RV p (IA p (4 * (g + 1)))) := by have hck' := hck simp only [gChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, decide_eq_true_eq] at hck' - obtain ⟨⟨⟨⟨⟨⟨⟨hsl, hrej⟩, hin⟩, _⟩, _⟩, _⟩, _⟩, _⟩ := hck' - have hT : ∀ σ s, IA p r c σ s → T p σ s := fun _ _ h => h.t - have hTG : ∀ σ s, (∃ h, HN p σ h ∧ GS p h r c c₀ 4 σ s) → T p σ s := fun _ _ ⟨_, _, g⟩ => g.s3.t - refine relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (aGrp_ok C hp hv hr hck hs) fun _ h' => ⟨h, hh, h'⟩) ?_ + obtain ⟨⟨⟨⟨⟨hsl, hrej⟩, hin⟩, _⟩, _⟩, hl⟩ := hck' + have hTG : ∀ σ s, (∃ h, HN p σ h ∧ GS p h (4 * g) 4 σ s) → T p σ s := fun _ _ ⟨_, _, g⟩ => g.s3.t + refine relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (aGrp_ok C hp hv hck hs) fun _ h' => ⟨h, hh, h'⟩) ?_ unfold aGrp - refine RelCT.seq (RelCT.mono (slot_tr hp (by omega) (by decide) (by omega) (hsl 0 (by decide))) + refine RelCT.seq (RelCT.mono (slot_tr hp (by decide) (by omega) (hsl 0 (by decide))) (fun x y ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁⟩, ⟨h₂, hh₂, s₂⟩⟩ => ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁, fun _ h => absurd h (by omega)⟩, ⟨h₂, hh₂, s₂, fun _ h => absurd h (by omega)⟩⟩) fun _ _ h => h) ?_ - refine RelCT.seq (slot_tr hp (by omega) (by decide) (by omega) (hsl 1 (by decide))) ?_ - refine RelCT.seq (slot_tr hp (by omega) (by decide) (by omega) (hsl 2 (by decide))) ?_ - refine RelCT.seq (slot_tr hp (by omega) (by decide) (by omega) (hsl 3 (by decide))) ?_ + refine RelCT.seq (slot_tr hp (by decide) (by omega) (hsl 1 (by decide))) ?_ + refine RelCT.seq (slot_tr hp (by decide) (by omega) (hsl 2 (by decide))) ?_ + refine RelCT.seq (slot_tr hp (by decide) (by omega) (hsl 3 (by decide))) ?_ unfold sampled4 refine RelCT.seq (RelCT.sameB (rej4At_tr C.rej4 (layOk p hp) hrej fun x y h => ?_) (fun x y h => ?_) (fun x y h => (RV.lrel hp hTG h).2.2)) @@ -125,39 +123,12 @@ theorem aGrp_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r exact ⟨WP.mono (rej4At_ok C.rej4 L.1 hrej) fun _ h' => ⟨_, h'.1⟩, WP.mono (rej4At_ok C.rej4 L.2.1 hrej) fun _ h' => ⟨_, h'.1⟩⟩ -theorem IA.next {p : Params} {r : Nat} {x y : State} (h : RV p (IA p r p.ℓ) x y) : RV p (IA p (r + 1) 0) x y := by - obtain ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁⟩, ⟨h₂, hh₂, s₂⟩⟩ := h - exact ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, s₁.next⟩, ⟨h₂, hh₂, s₂.next⟩⟩ - -theorem aRow_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {r : Nat} (hr : r < p.k) : - RelCT isa (RV p (IA p r 0)) (aRow P p r) (RV p (IA p (r + 1) 0)) := by - have hkl := kl_le p hp - have hg := gChk_all p hp r hr - have hl : p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by - have : ∀ p ∈ params, p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by decide - exact this p hp - unfold aRow - refine RelCT.seq (aGrp_tr C hp hr hg.1) ?_ - by_cases h7 : p.ℓ = 7 - · rw [ite_eq_left h7] - refine RelCT.mono (aGrp_tr C hp hr (hg.2 h7)) (fun x y h => by rwa [Nat.zero_add] at h) fun x y h => ?_ - rw [show 3 + 4 = p.ℓ by omega] at h - exact IA.next h - · rw [ite_eq_right h7] - refine RelCT.mono (seqR_tr (R := fun e => RV p (IA p r (e - 8 * r))) (p.ℓ - 4) (8 * r + 4) fun e he he' => ?_) - (fun x y h => by rw [show 8 * r + 4 - 8 * r = 4 by omega]; rwa [Nat.zero_add] at h) fun x y h => ?_ - · have := aOne_tr C hp hr (c := e - 8 * r) (by omega) - rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this - exact this - · rw [show 8 * r + 4 + (p.ℓ - 4) - 8 * r = p.ℓ by omega] at h - exact IA.next h - theorem ballStage_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) : - RelCT isa (RV p (IA p p.k 0)) (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC) (RV p (I4 p)) := by + RelCT isa (RV p (IA p (p.k * p.ℓ))) (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC) (RV p (I4 p)) := by have hc := sChk_all p hp simp only [sChk, Bool.and_eq_true, decide_eq_true_eq, List.all_eq_true, List.mem_range] at hc obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨_, _⟩, _⟩, _⟩, c5⟩, c6⟩, c7⟩, _⟩, c9⟩, _⟩, _⟩ := hc - have hT : ∀ σ s, IA p p.k 0 σ s → T p σ s := fun _ _ h => h.t + have hT : ∀ σ s, IA p (p.k * p.ℓ) σ s → T p σ s := fun _ _ h => h.t refine relInv (fun σ s hv ⟨h, hh, hs⟩ => WP.mono (ballStage_ok C hp hv hs) fun _ h' => ⟨h, hh, h'⟩) ?_ unfold sampled refine RelCT.seq (RelCT.sameB (ballAt_tr C.ball (layOk p hp) c6 c5 fun x y h => ?_) (fun x y h => ?_) @@ -203,10 +174,15 @@ theorem samples_tr {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) (RV.lrel hp (fun _ _ h => let ⟨_, _, hs, _⟩ := h; hs.t) h).2.2)) (RelCT.seq (copyK_tr hp (j := 0) (by decide)) (RelCT.seq (copyK_tr hp (j := 1) (by decide)) (RelCT.seq (copyK_tr hp (j := 2) (by decide)) (RelCT.mono (copyK_tr hp (j := 3) (by decide)) (fun _ _ h => h) - (Q' := RV p (IA p 0 0)) fun x y h => ?_))))) (RelCT.seq ?_ (ballStage_tr C hp)) + (Q' := RV p (IA p 0)) fun x y h => ?_))))) (RelCT.seq (R := RV p (IA p (4 * (p.k * p.ℓ / 4)))) ?_ + (RelCT.seq (R := RV p (IA p (p.k * p.ℓ))) ?_ (ballStage_tr C hp))) · obtain ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, r₁⟩, ⟨h₂, hh₂, r₂⟩⟩ := h exact ⟨σ₁, σ₂, v₁, v₂, pub, ⟨h₁, hh₁, r₁.s3⟩, ⟨h₂, hh₂, r₂.s3⟩⟩ - have := seqR_tr (R := fun r => RV p (IA p r 0)) p.k 0 fun r _ hr => aRow_tr C hp (by omega) - rwa [Nat.zero_add] at this + · have := seqR_tr (R := fun g => RV p (IA p (4 * g))) (p.k * p.ℓ / 4) 0 fun g _ hg => + aGrp_tr C hp (gChk_all p hp g (by omega)) + rwa [Nat.zero_add] at this + · have := seqR_tr (R := fun e => RV p (IA p e)) (p.k * p.ℓ % 4) (4 * (p.k * p.ℓ / 4)) fun e _ he => + aOne_tr C hp (by omega) + rwa [Nat.div_add_mod] at this end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Correct.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Correct.lean index d3924f5fd..1010cc235 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Correct.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Correct.lean @@ -49,7 +49,7 @@ theorem S2.flag {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) { theorem S4.toSC {p : Params} {h : List (Vector Bool n)} {σ s : State} (hs : S4 p h σ s) {q : Bool} (h15 : s.gpr .r15 = flag (q = true)) : - SC p h (fun r c => polyAt s.mem (pa s (pA r c))) (q = true) 0 (polyAt s.mem (pa s pC)) 0 σ s := + SC p h (fun r c => polyAt s.mem (pa s (pA p.ℓ r c))) (q = true) 0 (polyAt s.mem (pa s pC)) 0 σ s := ⟨hs.t, hs.hint, fun r hr c hc => ⟨hs.red r hr c hc, rfl⟩, fun i hi => by rw [iteN (Nat.not_lt_zero _)]; exact hs.z i hi, ⟨hs.redC, rfl⟩, fun _ h => absurd h (Nat.not_lt_zero _), h15⟩ @@ -73,9 +73,9 @@ theorem cs_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : | true => obtain ⟨hA, hB⟩ := hok rfl obtain ⟨nA, hnA⟩ := common_bound (P := fun r n => ∀ c < p.ℓ, - rejNTTPoly n (aSeed (vPk p σ) r c) = some (polyAt s.mem (pa s (pA r c)))) + rejNTTPoly n (aSeed (vPk p σ) r c) = some (polyAt s.mem (pa s (pA p.ℓ r c)))) (fun _ _ _ hle h c hc => rejNTTPoly_mono hle (h c hc)) p.k fun r hr => - common_bound (P := fun c n => rejNTTPoly n (aSeed (vPk p σ) r c) = some (polyAt s.mem (pa s (pA r c)))) + common_bound (P := fun c n => rejNTTPoly n (aSeed (vPk p σ) r c) = some (polyAt s.mem (pa s (pA p.ℓ r c)))) (fun _ _ _ hle h => rejNTTPoly_mono hle h) p.ℓ fun c hc => let ⟨b, hb⟩ := hA r hr c hc; ⟨b.rejNTT, hb⟩ obtain ⟨bB, hbB⟩ := hB @@ -84,7 +84,7 @@ theorem cs_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : have e := verifyMu_rows p ⟨0, 0, nA, bB.ball⟩ (vPk p σ) (vMu σ) (vSig p σ) hh hl hnA hcc obtain ⟨hg, hβ, _, _⟩ := parChk p hp rw [decide_eq_true ((normR_vZ_iff hβ p hg _).mpr hn), hcc', Bool.true_and] at e - by_cases hE : H (vMu σ ++ w1Enc p σ h (fun r c => polyAt s.mem (pa s (pA r c))) (ntt (polyAt s.mem (pa s pC)))) + by_cases hE : H (vMu σ ++ w1Enc p σ h (fun r c => polyAt s.mem (pa s (pA p.ℓ r c))) (ntt (polyAt s.mem (pa s pC)))) p.ctildeLen = vCt p (vSig p σ) · exact .inl ⟨by rw [h15']; exact flag_congr (by simp [hE]), ⟨_, e.trans (congrArg some (beq_iff_eq.mpr hE.symm))⟩⟩ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean index 4cdc223c9..8334da579 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Instrs.lean @@ -79,21 +79,15 @@ theorem hint_q : SameQ q (hint P p) (hint P0 p) := (SameQ.call hP.hintUnpack _). theorem zOne_q (i : Nat) : SameQ q (zOne P p i) (zOne P0 p i) := (SameQ.call hP.bitUnpack _).seq ((SameQ.call hP.normLt _).seq rfl) -omit p in -theorem aOne_q (e : Nat) : SameQ q (aOne P e) (aOne P0 e) := +theorem aOne_q (e : Nat) : SameQ q (aOne P p e) (aOne P0 p e) := SameQ.seq rfl (SameQ.sampled (SameQ.call hP.rejNtt _) _) -theorem aGrp_q (r c : Nat) : SameQ q (aGrp P p r c) (aGrp P0 p r c) := +theorem aGrp_q (g : Nat) : SameQ q (aGrp P p g) (aGrp P0 p g) := SameQ.seq rfl (SameQ.seq rfl (SameQ.seq rfl (SameQ.seq rfl (SameQ.sampled4 (SameQ.call hP.rej4 _) _)))) -theorem aRow_q (r : Nat) : SameQ q (aRow P p r) (aRow P0 p r) := by - refine (aGrp_q hP p r 0).seq ?_ - by_cases h : p.ℓ = 7 - · rw [ite_eq_left h, ite_eq_left h]; exact aGrp_q hP p r 3 - · rw [ite_eq_right h, ite_eq_right h]; exact SameQ.seqR (aOne_q hP) _ _ - theorem samples_q : SameQ q (samples P p) (samples P0 p) := - SameQ.seq rfl ((SameQ.seqR (aRow_q hP p) _ _).seq (SameQ.sampled (SameQ.call hP.ball _) _)) + SameQ.seq rfl ((SameQ.seqR (aGrp_q hP p) _ _).seq ((SameQ.seqR (aOne_q hP p) _ _).seq + (SameQ.sampled (SameQ.call hP.ball _) _))) theorem dot_q (r : Nat) : SameQ q (dot P p r) (dot P0 p r) := (SameQ.call hP.mul _).seq (SameQ.seqR (fun _ => SameQ.call hP.mulAdd _) _ _) @@ -167,7 +161,7 @@ variable {P : Prims} (hP : PrimsC P) (p : Params) include hP theorem verify_c : SameC (verify P p) (verify P0 p) := by - have aOne : ∀ e, SameC (aOne P e) (aOne P0 e) := fun e => + have aOne : ∀ e, SameC (aOne P p e) (aOne P0 p e) := fun e => SameC.seq rfl (SameC.sampled (SameC.call hP.rejNtt _) _) have dot : ∀ r, SameC (dot P p r) (dot P0 p r) := fun r => (SameC.call hP.mul _).seq (SameC.seqR (fun _ => SameC.call hP.mulAdd _) _ _) @@ -175,16 +169,11 @@ theorem verify_c : SameC (verify P p) (verify P0 p) := by (dot r).seq ((SameC.call hP.unpackT1 _).seq ((SameC.call hP.ntt _).seq ((SameC.call hP.mul _).seq ((SameC.call hP.sub _).seq ((SameC.call hP.invNtt _).seq ((SameC.call hP.useHint _).seq (SameC.call hP.simpleBitPack _))))))) - have aGrp : ∀ r c, SameC (aGrp P p r c) (aGrp P0 p r c) := fun r c => + have aGrp : ∀ g, SameC (aGrp P p g) (aGrp P0 p g) := fun g => SameC.seq rfl (SameC.seq rfl (SameC.seq rfl (SameC.seq rfl (SameC.sampled4 (SameC.call hP.rej4 _) _)))) - have aRow : ∀ r, SameC (aRow P p r) (aRow P0 p r) := fun r => by - refine (aGrp r 0).seq ?_ - by_cases h : p.ℓ = 7 - · rw [ite_eq_left h, ite_eq_left h]; exact aGrp r 3 - · rw [ite_eq_right h, ite_eq_right h]; exact SameC.seqR aOne _ _ have samples : SameC (samples P p) (samples P0 p) := - SameC.seq rfl ((SameC.seqR aRow _ _).seq - (SameC.sampled (SameC.call hP.ball _) _)) + SameC.seq rfl ((SameC.seqR aGrp _ _).seq ((SameC.seqR aOne _ _).seq + (SameC.sampled (SameC.call hP.ball _) _))) have compute : SameC (compute P p) (compute P0 p) := (SameC.seqR (fun _ => SameC.call hP.ntt _) _ _).seq ((SameC.call hP.ntt _).seq ((SameC.seqR row _ _).seq rfl)) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean index a84e85aef..ee43c7308 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageA.lean @@ -3,9 +3,10 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Verify.StageZ /-! # ML-DSA verification on x86-64: the samplers -`ρ` to `SB`; each entry `Â[r, s]` sampled from `ρ ‖ s ‖ r` (`aOne_ok`), and -`c` (`ball_ok`), each reduced, and `r15` 1 only if every sampler succeeded, -with their outputs (`S3`, `S4`). +`ρ` to `SB` and `SB4`; each entry `Â[r, s]` (number `ℓr + s`) sampled from +`ρ ‖ s ‖ r`, four at a time (`aGrp_ok`) and then one at a time (`aOne_ok`), +and `c` (`ballStage_ok`), each reduced, and `r15` 1 only if every sampler +succeeded, with their outputs (`S3`, `S4`). -/ namespace VG.Proof.MlDsa.X86_64.Verify @@ -40,21 +41,44 @@ theorem integerToBytes_one (x : Nat) : integerToBytes x 1 = [BitVec.ofNat 8 x] : /-! ## `Â` -/ -/-- Entry `(r', c')` of `Â` is sampled before entry `(r, c)`. -/ -def Done (r c r' c' : Nat) : Prop := r' < r ∨ (r' = r ∧ c' < c) +/-- Entry `(r', c')` of `Â`, number `l r' + c'` in rows of `l` entries, is sampled before entry `e`. -/ +def Done (l e r' c' : Nat) : Prop := l * r' + c' < e -/-- After the entries of `Â` before `(r, c)`, with the hint `h`. -/ -structure S3 (p : Params) (h : List (Vector Bool n)) (r c : Nat) (σ st : State) : Prop where +theorem rc_eq {l r c e : Nat} (hc : c < l) (h : l * r + c = e) : r = e / l ∧ c = e % l := by + have hl : 0 < l := by omega + subst h + constructor + · rw [Nat.add_comm, Nat.add_mul_div_left _ _ hl, Nat.div_eq_of_lt hc, Nat.zero_add] + · rw [Nat.add_comm, Nat.add_mul_mod_self_left, Nat.mod_eq_of_lt hc] + +theorem pA_eq (l r c : Nat) : pA l r c = pS (20 + (l * r + c)) := by + show pS (20 + l * r + c) = _ + rw [Nat.add_assoc] + +theorem l_pos : ∀ p ∈ params, 0 < p.ℓ := by decide + +/-- Entry `e < kℓ` is `Â[e / ℓ, e mod ℓ]`. -/ +theorem entry_lt {p : Params} (hp : p ∈ params) {e : Nat} (he : e < p.k * p.ℓ) : e / p.ℓ < p.k ∧ e % p.ℓ < p.ℓ := + ⟨(Nat.div_lt_iff_lt_mul (l_pos p hp)).mpr he, Nat.mod_lt _ (l_pos p hp)⟩ + +theorem done_all {p : Params} {r c : Nat} (hr : r < p.k) (hc : c < p.ℓ) : Done p.ℓ (p.k * p.ℓ) r c := by + unfold Done + have := Nat.mul_le_mul_left p.ℓ (show r + 1 ≤ p.k by omega) + rw [Nat.mul_add, Nat.mul_one, Nat.mul_comm p.ℓ p.k] at this + omega + +/-- After the first `e` entries of `Â`, with the hint `h`. -/ +structure S3 (p : Params) (h : List (Vector Bool n)) (e : Nat) (σ st : State) : Prop where t : T p σ st hint : HintIs st.mem (pa st (pH 0)) p.k h z : ∀ i < p.ℓ, PolyIs st.mem (pa st (pZ i)) (toRq (vZ p (vSig p σ) i)) rho : bytesAt st.mem (pa st (sc oSB)) 32 = vRho (vPk p σ) rho4 : ∀ k < 4, bytesAt st.mem (pa st (sc (oSB4 + 34 * k))) 32 = vRho (vPk p σ) - red : ∀ r' < p.k, ∀ c' < p.ℓ, Done r c r' c' → Reduced st.mem (pa st (pA r' c')) + red : ∀ r' < p.k, ∀ c' < p.ℓ, Done p.ℓ e r' c' → Reduced st.mem (pa st (pA p.ℓ r' c')) ok : ∃ q : Bool, st.gpr .r15 = flag (q = true) ∧ - (q = true → ∀ r' < p.k, ∀ c' < p.ℓ, Done r c r' c' → - ∃ b : Bounds, rejNTTPoly b.rejNTT (aSeed (vPk p σ) r' c') = some (polyAt st.mem (pa st (pA r' c')))) ∧ - (q = false → ∃ r' < p.k, ∃ c' < p.ℓ, Done r c r' c' ∧ rejNTTPoly minBounds.rejNTT (aSeed (vPk p σ) r' c') = none) + (q = true → ∀ r' < p.k, ∀ c' < p.ℓ, Done p.ℓ e r' c' → + ∃ b : Bounds, rejNTTPoly b.rejNTT (aSeed (vPk p σ) r' c') = some (polyAt st.mem (pa st (pA p.ℓ r' c')))) ∧ + (q = false → ∃ r' < p.k, ∃ c' < p.ℓ, Done p.ℓ e r' c' ∧ rejNTTPoly minBounds.rejNTT (aSeed (vPk p σ) r' c') = none) abbrev wsB : List (Ptr × Nat) := [(sc (oSB + 32), 1), (sc (oSB + 33), 1)] abbrev wsA (e : Nat) : List (Ptr × Nat) := [(pS (20 + e), 1024), (sc oSS, 2048)] @@ -63,17 +87,17 @@ abbrev wsA (e : Nat) : List (Ptr × Nat) := [(pS (20 + e), 1024), (sc oSS, 2048) def keepChk (p : Params) (ws : List (Ptr × Nat)) : Bool := tChk p ws && keepB (vB p) ws (pH 0) (1024 * p.k) && (List.range p.ℓ).all (fun i => keepB (vB p) ws (pZ i) 1024) -/-- The facts about the parameters entry `e = 8r + c` needs. -/ +/-- The facts about the parameters entry `e` needs. -/ def aChk (p : Params) (e : Nat) : Bool := inB (vW p) (sc (oSB + 32)) 1 && inB (vW p) (sc (oSB + 33)) 1 && rejChk (vB p) (vW p) (pS (20 + e)) && inB (vB p) (pS (20 + e)) 1024 && inB (vW p) (pS (20 + e)) 1024 && keepChk p wsB && keepChk p (wsA e) && keepB (vB p) wsB (sc oSB) 32 && keepB (vB p) (wsA e) (sc oSB) 32 && - (List.range p.k).all (fun r' => (List.range p.ℓ).all fun c' => !decide (8 * r' + c' < e) || - (keepB (vB p) wsB (pS (20 + 8 * r' + c')) 1024 && keepB (vB p) (wsA e) (pS (20 + 8 * r' + c')) 1024)) && + (List.range p.k).all (fun r' => (List.range p.ℓ).all fun c' => !decide (p.ℓ * r' + c' < e) || + (keepB (vB p) wsB (pA p.ℓ r' c') 1024 && keepB (vB p) (wsA e) (pA p.ℓ r' c') 1024)) && (List.range 4).all (fun k => keepB (vB p) wsB (sc (oSB4 + 34 * k)) 32 && keepB (vB p) (wsA e) (sc (oSB4 + 34 * k)) 32) -theorem aChk_all : ∀ p ∈ params, ∀ r < p.k, ∀ c < p.ℓ, aChk p (8 * r + c) = true := by decide +theorem aChk_all : ∀ p ∈ params, ∀ e < p.k * p.ℓ, aChk p e = true := by decide theorem keepChk_spec {p : Params} {ws : List (Ptr × Nat)} (h : keepChk p ws = true) : @@ -114,60 +138,60 @@ theorem setB2_ok {p : Params} {s : State} (L : Lay (vR p) (vW p) s) (h32 : inB ( theorem kl_le : ∀ p ∈ params, p.ℓ ≤ 7 ∧ p.k ≤ 8 := by decide -theorem done_succ {r c r' c' : Nat} : Done r (c + 1) r' c' ↔ Done r c r' c' ∨ (r' = r ∧ c' = c) := by +theorem done_succ {l e r' c' : Nat} : Done l (e + 1) r' c' ↔ Done l e r' c' ∨ l * r' + c' = e := by unfold Done; omega theorem aOne_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) - {h : List (Vector Bool n)} {r c : Nat} (hr : r < p.k) (hc : c < p.ℓ) {s : State} (hs : S3 p h r c σ s) : - WP isa (aOne P (8 * r + c)) s (S3 p h r (c + 1) σ) := by - have hck := aChk_all p hp r hr c hc + {h : List (Vector Bool n)} {e : Nat} (he : e < p.k * p.ℓ) {s : State} (hs : S3 p h e σ s) : + WP isa (aOne P p e) s (S3 p h (e + 1) σ) := by + have hck := aChk_all p hp e he have hkl := kl_le p hp - have hc8 : c < 8 := by omega + obtain ⟨hr, hc⟩ := entry_lt hp he simp only [aChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, Bool.or_eq_true, Bool.not_eq_true', decide_eq_false_iff_not, Nat.not_lt] at hck obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨h32, h33⟩, hrej⟩, hin⟩, hwa⟩, kB⟩, kA⟩, kSB⟩, kSA⟩, kE⟩, k4⟩ := hck obtain ⟨tB, hB, zB⟩ := keepChk_spec kB obtain ⟨tA, hA, zA⟩ := keepChk_spec kA have L := hs.t.lay hp hv - have em : (8 * r + c) % 8 = c := by omega - have ed : (8 * r + c) / 8 = r := by omega unfold aOne - rw [em, ed] - refine WP.seq (WP.mono (setB2_ok L h32 h33 c r (by omega) (by omega)) fun s₁ ⟨hP₁, h15₁, hm₁⟩ => ?_) + refine WP.seq (WP.mono (setB2_ok L h32 h33 (e % p.ℓ) (e / p.ℓ) (by omega) (by omega)) + fun s₁ ⟨hP₁, h15₁, hm₁⟩ => ?_) have t₁ := hs.t.step hp hv hP₁ tB have L₁ := t₁.lay hp hv - have hseed : bytesAt s₁.mem (pa s₁ (sc oSB)) 34 = aSeed (vPk p σ) r c := by + have hseed : bytesAt s₁.mem (pa s₁ (sc oSB)) 34 = aSeed (vPk p σ) (e / p.ℓ) (e % p.ℓ) := by rw [hP₁.pa (by decide), hm₁, seed_bytes, hs.rho, aSeed, integerToBytes_one, integerToBytes_one] obtain ⟨q, h15, hok, hbad⟩ := hs.ok refine WP.mono (sampled_ok L₁ hin hwa (h15₁.trans h15) (List.mem_cons_self ..) (WP.mono (rejNttAt_ok C.rejNtt L₁ hrej) fun s' ⟨hP, h15', hr', ho⟩ => ⟨hP, h15', hr', ho⟩)) fun s₂ ⟨hP₂, hred, rr, hrr, h15₂, hs1, hs0⟩ => ?_ rw [hseed] at hs1 hs0 - have hPA : pa s₂ (pA r c) = pa s₁ (pS (20 + (8 * r + c))) := by - rw [hP₂.pa (show Reg.rbx ∈ bases by decide), pA, show 20 + 8 * r + c = 20 + (8 * r + c) by omega] + have hPA : ∀ r' c', c' < p.ℓ → p.ℓ * r' + c' = e → + pa s₂ (pA p.ℓ r' c') = pa s₁ (pS (20 + e)) ∧ r' = e / p.ℓ ∧ c' = e % p.ℓ := fun r' c' hc' he' => + ⟨by rw [hP₂.pa (show Reg.rbx ∈ bases by decide), pA_eq, he'], rc_eq hc' he'⟩ refine ⟨t₁.step hp hv hP₂ tA, L₁.keepHint hP₂ hA (L.keepHint hP₁ hB hs.hint), fun i hi => L₁.keepPoly hP₂ (zA i hi) (L.keepPoly hP₁ (zB i hi) (hs.z i hi)), by rw [L₁.keepBytes hP₂ kSA, L.keepBytes hP₁ kSB, hs.rho], fun k hk => by rw [L₁.keepBytes hP₂ (k4 k hk).2, L.keepBytes hP₁ (k4 k hk).1, hs.rho4 k hk], fun r' hr' c' hc' hd => ?_, ⟨q && (rr == 1), ?_, fun hq r' hr' c' hc' hd => ?_, fun hq => ?_⟩⟩ - · rcases done_succ.mp hd with hd | ⟨rfl, rfl⟩ + · rcases done_succ.mp hd with hd | he' · have := kE r' hr' c' hc' - have hlt : 8 * r' + c' < 8 * r + c := by unfold Done at hd; omega - rw [or_iff_right (by omega)] at this + rw [or_iff_right (by unfold Done at hd; omega)] at this exact L₁.keepRed hP₂ this.2 (L.keepRed hP₁ this.1 (hs.red r' hr' c' hc' hd)) - · rw [hPA]; exact hred + · rw [(hPA r' c' hc' he').1]; exact hred · rw [h15₂] exact flag_congr (by cases q <;> simp) · simp only [Bool.and_eq_true, beq_iff_eq] at hq - rcases done_succ.mp hd with hd | ⟨rfl, rfl⟩ + rcases done_succ.mp hd with hd | he' · have := kE r' hr' c' hc' - have hlt : 8 * r' + c' < 8 * r + c := by unfold Done at hd; omega - rw [or_iff_right (by omega)] at this + rw [or_iff_right (by unfold Done at hd; omega)] at this obtain ⟨b, hb⟩ := hok hq.1 r' hr' c' hc' hd exact ⟨b, by rw [hb, L₁.keepPolyAt hP₂ this.2, L.keepPolyAt hP₁ this.1]⟩ - · obtain ⟨b, hb⟩ := hs1 hq.2 - exact ⟨b, by rw [hb, hPA]⟩ + · obtain ⟨e1, e2, e3⟩ := hPA r' c' hc' he' + obtain ⟨b, hb⟩ := hs1 hq.2 + refine ⟨b, ?_⟩ + rw [e1, e2, e3] + exact hb · cases hq' : q · obtain ⟨r', hr', c', hc', hd, hn⟩ := hbad hq' exact ⟨r', hr', c', hc', done_succ.mpr (.inl hd), hn⟩ @@ -176,55 +200,32 @@ theorem aOne_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ rcases hrr with h1 | h0 · rw [h1] at hq; cases hq · exact h0 - exact ⟨r, hr, c, hc, done_succ.mpr (.inr ⟨rfl, rfl⟩), hs0 h0⟩ - -end VG.Proof.MlDsa.X86_64.Verify - -namespace VG.Proof.MlDsa.X86_64.Verify - -open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Verify -open VG.Proof.MlKem.X86_64 -open VG.Spec.MlDsa -open VG.Spec.Sha3 (bytesAt) -open VG.Proof.MlDsa.Verify (vHint vZ vCt vRho aSeed) - -theorem S3.next {p : Params} {h : List (Vector Bool n)} {r : Nat} {σ s : State} (hs : S3 p h r p.ℓ σ s) : - S3 p h (r + 1) 0 σ s := by - have e : ∀ r' c', c' < p.ℓ → (Done r p.ℓ r' c' ↔ Done (r + 1) 0 r' c') := fun r' c' hc => by - unfold Done; omega - obtain ⟨q, h15, hok, hbad⟩ := hs.ok - refine ⟨hs.t, hs.hint, hs.z, hs.rho, hs.rho4, fun r' hr' c' hc' hd => hs.red r' hr' c' hc' ((e r' c' hc').mpr hd), - q, h15, fun hq r' hr' c' hc' hd => hok hq r' hr' c' hc' ((e r' c' hc').mpr hd), fun hq => ?_⟩ - obtain ⟨r', hr', c', hc', hd, hn⟩ := hbad hq - exact ⟨r', hr', c', hc', (e r' c' hc').mp hd, hn⟩ + exact ⟨e / p.ℓ, hr, e % p.ℓ, hc, done_succ.mpr (.inr (Nat.div_add_mod e p.ℓ)), hs0 h0⟩ /-! ## Four entries at a time -/ -/-- A piece writing `ws` keeps the entries done before `(r, c)` but those `ex` says, and `S3`'s other facts. -/ -def s3Chk (p : Params) (r c : Nat) (ex : Nat → Nat → Bool) (ws : List (Ptr × Nat)) : Bool := +/-- A piece writing `ws` keeps the entries done before `e` but those `ex` says, and `S3`'s other facts. -/ +def s3Chk (p : Params) (e : Nat) (ex : Nat → Nat → Bool) (ws : List (Ptr × Nat)) : Bool := keepChk p ws && keepB (vB p) ws (sc oSB) 32 && (List.range 4).all (fun k => keepB (vB p) ws (sc (oSB4 + 34 * k)) 32) && (List.range p.k).all (fun r' => (List.range p.ℓ).all fun c' => - !(decide (r' < r) || (r' == r && decide (c' < c))) || ex r' c' || keepB (vB p) ws (pA r' c') 1024) + !decide (p.ℓ * r' + c' < e) || ex r' c' || keepB (vB p) ws (pA p.ℓ r' c') 1024) -theorem s3Chk_spec {p : Params} {r c : Nat} {ex : Nat → Nat → Bool} {ws : List (Ptr × Nat)} - (h : s3Chk p r c ex ws = true) : +theorem s3Chk_spec {p : Params} {e : Nat} {ex : Nat → Nat → Bool} {ws : List (Ptr × Nat)} + (h : s3Chk p e ex ws = true) : keepChk p ws = true ∧ keepB (vB p) ws (sc oSB) 32 = true ∧ (∀ k < 4, keepB (vB p) ws (sc (oSB4 + 34 * k)) 32 = true) ∧ - ∀ r' < p.k, ∀ c' < p.ℓ, Done r c r' c' → ex r' c' = false → keepB (vB p) ws (pA r' c') 1024 = true := by + ∀ r' < p.k, ∀ c' < p.ℓ, Done p.ℓ e r' c' → ex r' c' = false → keepB (vB p) ws (pA p.ℓ r' c') 1024 = true := by simp only [s3Chk, Bool.and_eq_true] at h obtain ⟨⟨⟨h1, h2⟩, h3⟩, h4⟩ := h refine ⟨h1, h2, fun k hk => List.all_eq_true.mp h3 k (List.mem_range.mpr hk), fun r' hr' c' hc' hd hx => ?_⟩ have := List.all_eq_true.mp (List.all_eq_true.mp h4 r' (List.mem_range.mpr hr')) c' (List.mem_range.mpr hc') - have hd' : (decide (r' < r) || (r' == r && decide (c' < c))) = true := by - unfold Done at hd - rcases hd with h | ⟨rfl, h⟩ <;> simp [h] - rw [hd', hx] at this + rw [decide_eq_true (show p.ℓ * r' + c' < e from hd), hx] at this simpa using this /-- `S3` across a piece that writes `ws`, which `s3Chk` says keeps it. -/ -theorem S3.keep {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {r c : Nat} - {s s' : State} (hs : S3 p h r c σ s) {ws : List (Ptr × Nat)} (hP : PPostB s s' ws) - (h15 : s'.gpr .r15 = s.gpr .r15) (hc : s3Chk p r c (fun _ _ => false) ws = true) : S3 p h r c σ s' := by +theorem S3.keep {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {e : Nat} + {s s' : State} (hs : S3 p h e σ s) {ws : List (Ptr × Nat)} (hP : PPostB s s' ws) + (h15 : s'.gpr .r15 = s.gpr .r15) (hc : s3Chk p e (fun _ _ => false) ws = true) : S3 p h e σ s' := by obtain ⟨hk, kSB, k4, kE⟩ := s3Chk_spec hc obtain ⟨tk, hh, zk⟩ := keepChk_spec hk have L := hs.t.lay hp hv @@ -239,158 +240,130 @@ theorem S3.keep {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) { /-- The two bytes of seed `j` of `SB4`. -/ abbrev wsS (j : Nat) : List (Ptr × Nat) := [(sc (oSB4 + 34 * j + 32), 1), (sc (oSB4 + 34 * j + 33), 1)] -/-- What setting the bytes of seed `j` needs, after `(r, c)`. -/ -def slotChk (p : Params) (r c j : Nat) : Bool := +/-- What setting the bytes of seed `j` needs, after `e` entries. -/ +def slotChk (p : Params) (e j : Nat) : Bool := inB (vW p) (sc (oSB4 + 34 * j + 32)) 1 && inB (vW p) (sc (oSB4 + 34 * j + 33)) 1 && - s3Chk p r c (fun _ _ => false) (wsS j) && + s3Chk p e (fun _ _ => false) (wsS j) && (List.range j).all (fun k => keepB (vB p) (wsS j) (sc (oSB4 + 34 * k)) 34) -/-- After the bytes of the first `j` seeds of `SB4` for the entries `(r, c₀ + k)`. -/ -structure GS (p : Params) (h : List (Vector Bool n)) (r c c₀ j : Nat) (σ st : State) : Prop where - s3 : S3 p h r c σ st - done : ∀ k < j, bytesAt st.mem (pa st (sc (oSB4 + 34 * k))) 34 = aSeed (vPk p σ) r (c₀ + k) +/-- After the bytes of the first `j` seeds of `SB4`, for the entries `e + k`. -/ +structure GS (p : Params) (h : List (Vector Bool n)) (e j : Nat) (σ st : State) : Prop where + s3 : S3 p h e σ st + done : ∀ k < j, bytesAt st.mem (pa st (sc (oSB4 + 34 * k))) 34 = aSeed (vPk p σ) ((e + k) / p.ℓ) ((e + k) % p.ℓ) theorem slot_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} - {r c c₀ j : Nat} (hr : r < 256) (hj : j < 4) (hx : c₀ + j < 256) (hck : slotChk p r c j = true) {s : State} - (hs : GS p h r c c₀ j σ s) : WP isa (.block (setSR r c₀ j)) s (GS p h r c c₀ (j + 1) σ) := by + {e j : Nat} (hj : j < 4) (he : e + j < p.k * p.ℓ) (hck : slotChk p e j = true) {s : State} + (hs : GS p h e j σ s) : WP isa (.block (setSR p e j)) s (GS p h e (j + 1) σ) := by + have hkl := kl_le p hp + obtain ⟨hr, hc⟩ := entry_lt hp he simp only [slotChk, Bool.and_eq_true, List.all_eq_true, List.mem_range] at hck obtain ⟨⟨⟨h32, h33⟩, h3⟩, hk⟩ := hck have L := hs.s3.t.lay hp hv unfold setSR - refine WP.mono (setB2At_ok L (oSB4 + 34 * j) h32 h33 (c₀ + j) r hx hr) fun s₁ ⟨hP₁, h15₁, hm₁⟩ => - ⟨hs.s3.keep hp hv hP₁ h15₁ h3, fun k hk' => ?_⟩ + refine WP.mono (setB2At_ok L (oSB4 + 34 * j) h32 h33 ((e + j) % p.ℓ) ((e + j) / p.ℓ) (by omega) (by omega)) + fun s₁ ⟨hP₁, h15₁, hm₁⟩ => ⟨hs.s3.keep hp hv hP₁ h15₁ h3, fun k hk' => ?_⟩ rcases (by omega : k < j ∨ k = j) with hk' | rfl · rw [L.keepBytes hP₁ (hk k hk'), hs.done k hk'] - · rw [hP₁.pa (show Reg.rbx ∈ bases by decide), hm₁, seed_bytes, hs.s3.rho4 k hj, aSeed, integerToBytes_one, integerToBytes_one] - + · rw [hP₁.pa (show Reg.rbx ∈ bases by decide), hm₁, seed_bytes, hs.s3.rho4 k hj, aSeed, integerToBytes_one, + integerToBytes_one] -/-- Whether `(r', c')` is one of the four entries from `(r, c₀)`. -/ -abbrev inGrp (r c₀ r' c' : Nat) : Bool := r' == r && decide (c₀ ≤ c') && decide (c' < c₀ + 4) +/-- Whether entry `(r', c')` is one of the four from `e`. -/ +abbrev inGrp (l e r' c' : Nat) : Bool := decide (e ≤ l * r' + c') && decide (l * r' + c' < e + 4) -/-- The facts about the parameters the entries `(r, c₀), …, (r, c₀ + 3)` need, after `(r, c)`. -/ -def gChk (p : Params) (r c₀ c : Nat) : Bool := - (List.range 4).all (fun j => slotChk p r c j) && rej4Chk (vB p) (vW p) (pA r c₀) (sc (oR4 p)) && - inB (vB p) (pA r c₀) 4096 && inB (vW p) (pA r c₀) 4096 && - s3Chk p r c (inGrp r c₀) [(pA r c₀, 4096), (sc (oR4 p), 8192)] && - decide (c₀ + 4 ≤ p.ℓ) && decide (c₀ ≤ c) && decide (c ≤ c₀ + 4) +/-- The facts about the parameters the entries `e, …, e + 3` need. -/ +def gChk (p : Params) (e : Nat) : Bool := + (List.range 4).all (fun j => slotChk p e j) && rej4Chk (vB p) (vW p) (pS (20 + e)) (sc (oR4 p)) && + inB (vB p) (pS (20 + e)) 4096 && inB (vW p) (pS (20 + e)) 4096 && + s3Chk p e (inGrp p.ℓ e) [(pS (20 + e), 4096), (sc (oR4 p), 8192)] && decide (e + 4 ≤ p.k * p.ℓ) -theorem gChk_all : ∀ p ∈ params, ∀ r < p.k, gChk p r 0 0 = true ∧ (p.ℓ = 7 → gChk p r 3 4 = true) := by decide +theorem gChk_all : ∀ p ∈ params, ∀ g < p.k * p.ℓ / 4, gChk p (4 * g) = true := by decide -theorem pa_poly4 (s : State) (r c k : Nat) : poly4 (pa s (pA r c)) k = pa s (pA r (c + k)) := by +theorem pa_poly4 (s : State) (e k : Nat) : poly4 (pa s (pS (20 + e))) k = pa s (pS (20 + (e + k))) := by unfold poly4 simp only [pa] - rw [BitVec.add_assoc, ← BitVec.ofNat_add, show oP (20 + 8 * r + c) + 1024 * k = oP (20 + 8 * r + (c + k)) by + rw [BitVec.add_assoc, ← BitVec.ofNat_add, show oP (20 + e) + 1024 * k = oP (20 + (e + k)) by simp only [oP]; omega] theorem aGrp_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) - {h : List (Vector Bool n)} {r c₀ c : Nat} (hr : r < p.k) (hck : gChk p r c₀ c = true) {s : State} - (hs : S3 p h r c σ s) : WP isa (aGrp P p r c₀) s (S3 p h r (c₀ + 4) σ) := by + {h : List (Vector Bool n)} {g : Nat} (hck : gChk p (4 * g) = true) {s : State} + (hs : S3 p h (4 * g) σ s) : WP isa (aGrp P p g) s (S3 p h (4 * (g + 1)) σ) := by have hkl := kl_le p hp simp only [gChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, decide_eq_true_eq] at hck - obtain ⟨⟨⟨⟨⟨⟨⟨hsl, hrej⟩, hin⟩, hwa⟩, hG⟩, hl⟩, hc₀⟩, hc4⟩ := hck + obtain ⟨⟨⟨⟨⟨hsl, hrej⟩, hin⟩, hwa⟩, hG⟩, hl⟩ := hck obtain ⟨hk, kSB, k4, kE⟩ := s3Chk_spec hG obtain ⟨tk, hh, zk⟩ := keepChk_spec hk unfold aGrp - refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 0 (by decide)) + refine WP.seq (WP.mono (slot_ok hp hv (by decide) (by omega) (hsl 0 (by decide)) ⟨hs, fun _ h => absurd h (by omega)⟩) fun _ g₁ => ?_) - refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 1 (by decide)) g₁) fun _ g₂ => ?_) - refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 2 (by decide)) g₂) fun _ g₃ => ?_) - refine WP.seq (WP.mono (slot_ok hp hv (by omega) (by decide) (by omega) (hsl 3 (by decide)) g₃) fun s₄ g₄ => ?_) + refine WP.seq (WP.mono (slot_ok hp hv (by decide) (by omega) (hsl 1 (by decide)) g₁) fun _ g₂ => ?_) + refine WP.seq (WP.mono (slot_ok hp hv (by decide) (by omega) (hsl 2 (by decide)) g₂) fun _ g₃ => ?_) + refine WP.seq (WP.mono (slot_ok hp hv (by decide) (by omega) (hsl 3 (by decide)) g₃) fun s₄ g₄ => ?_) have L₄ := g₄.s3.t.lay hp hv obtain ⟨q, h15, hok, hbad⟩ := g₄.s3.ok - have hseed : ∀ k < 4, seed4 s₄.mem (pa s₄ (sc oSB4)) k = aSeed (vPk p σ) r (c₀ + k) := fun k hk => by + have hseed : ∀ k < 4, seed4 s₄.mem (pa s₄ (sc oSB4)) k = + aSeed (vPk p σ) ((4 * g + k) / p.ℓ) ((4 * g + k) % p.ℓ) := fun k hk => by unfold seed4 rw [show pa s₄ (sc oSB4) + BitVec.ofNat 64 (34 * k) = pa s₄ (sc (oSB4 + 34 * k)) by simp only [pa]; rw [BitVec.add_assoc, ← BitVec.ofNat_add]] exact g₄.done k hk refine WP.mono (sampled4_ok L₄ hin hwa h15 (List.mem_cons_self ..) - (F := fun k b => rejNTTPoly b.rejNTT (aSeed (vPk p σ) r (c₀ + k))) + (F := fun k b => rejNTTPoly b.rejNTT (aSeed (vPk p σ) ((4 * g + k) / p.ℓ) ((4 * g + k) % p.ℓ))) (WP.mono (rej4At_ok C.rej4 L₄ hrej) fun s' ⟨hP, h15', hred, ho⟩ => ⟨hP, h15', hred, ?_⟩)) fun s₅ ⟨hP₅, hred₅, rr, hrr, h15₅, hs1, hs0⟩ => ?_ · rcases ho with ⟨h1, hb⟩ | ⟨h0, k, hk, hn⟩ · exact .inl ⟨h1, fun k hk => by rw [← hseed k hk]; exact hb k hk⟩ · exact .inr ⟨h0, k, hk, by rw [← hseed k hk]; exact hn⟩ - have e₅ : ∀ k, poly4 (pa s₄ (pA r c₀)) k = pa s₅ (pA r (c₀ + k)) := fun k => by + have e₅ : ∀ k, poly4 (pa s₄ (pS (20 + 4 * g))) k = pa s₅ (pS (20 + (4 * g + k))) := fun k => by rw [pa_poly4, hP₅.pa (show Reg.rbx ∈ bases by decide)] - have hgrp : ∀ c', inGrp r c₀ r c' = true → c₀ ≤ c' ∧ c' < c₀ + 4 := fun c' hg => by - simp only [inGrp, Bool.and_eq_true, beq_self_eq_true, decide_eq_true_eq, true_and] at hg; exact hg - have hout : ∀ r' c', Done r (c₀ + 4) r' c' → inGrp r c₀ r' c' = false → Done r c r' c' := fun r' c' hd hx => by - unfold Done at hd ⊢ - rcases hd with hd | ⟨rfl, hd⟩ - · exact .inl hd - · refine .inr ⟨rfl, ?_⟩ - simp only [inGrp, beq_self_eq_true, Bool.true_and, Bool.and_eq_false_iff, decide_eq_false_iff_not] at hx + have hgrp : ∀ r' c', inGrp p.ℓ (4 * g) r' c' = true → 4 * g ≤ p.ℓ * r' + c' ∧ p.ℓ * r' + c' < 4 * g + 4 := + fun r' c' hg => by simp only [inGrp, Bool.and_eq_true, decide_eq_true_eq] at hg; exact hg + have hout : ∀ r' c', Done p.ℓ (4 * (g + 1)) r' c' → inGrp p.ℓ (4 * g) r' c' = false → Done p.ℓ (4 * g) r' c' := + fun r' c' hd hx => by + simp only [inGrp, Bool.and_eq_false_iff, decide_eq_false_iff_not] at hx + unfold Done at hd ⊢ omega refine ⟨g₄.s3.t.step hp hv hP₅ tk, L₄.keepHint hP₅ hh g₄.s3.hint, fun i hi => L₄.keepPoly hP₅ (zk i hi) (g₄.s3.z i hi), by rw [L₄.keepBytes hP₅ kSB, g₄.s3.rho], fun k hk => by rw [L₄.keepBytes hP₅ (k4 k hk), g₄.s3.rho4 k hk], fun r' hr' c' hc' hd => ?_, q && (rr == 1), ?_, fun hq r' hr' c' hc' hd => ?_, fun hq => ?_⟩ - · cases hx : inGrp r c₀ r' c' + · cases hx : inGrp p.ℓ (4 * g) r' c' · exact L₄.keepRed hP₅ (kE r' hr' c' hc' (hout r' c' hd hx) hx) (g₄.s3.red r' hr' c' hc' (hout r' c' hd hx)) - · have e : r' = r := by simp only [inGrp, Bool.and_eq_true, beq_iff_eq] at hx; exact hx.1.1 - subst e - obtain ⟨g1, g2⟩ := hgrp c' hx - have := hred₅ (c' - c₀) (by omega) - rwa [e₅, show c₀ + (c' - c₀) = c' by omega] at this + · obtain ⟨g1, g2⟩ := hgrp r' c' hx + have := hred₅ (p.ℓ * r' + c' - 4 * g) (by omega) + rwa [e₅, show 4 * g + (p.ℓ * r' + c' - 4 * g) = p.ℓ * r' + c' by omega, ← pA_eq] at this · rw [h15₅] exact flag_congr (by cases q <;> simp) · simp only [Bool.and_eq_true, beq_iff_eq] at hq - cases hx : inGrp r c₀ r' c' + cases hx : inGrp p.ℓ (4 * g) r' c' · obtain ⟨b, hb⟩ := hok hq.1 r' hr' c' hc' (hout r' c' hd hx) exact ⟨b, by rw [hb, L₄.keepPolyAt hP₅ (kE r' hr' c' hc' (hout r' c' hd hx) hx)]⟩ - · have e : r' = r := by simp only [inGrp, Bool.and_eq_true, beq_iff_eq] at hx; exact hx.1.1 - subst e - obtain ⟨g1, g2⟩ := hgrp c' hx - obtain ⟨b, hb⟩ := hs1 hq.2 (c' - c₀) (by omega) - refine ⟨b, ?_⟩ - rwa [e₅, show c₀ + (c' - c₀) = c' by omega] at hb + · obtain ⟨g1, g2⟩ := hgrp r' c' hx + obtain ⟨b, hb⟩ := hs1 hq.2 (p.ℓ * r' + c' - 4 * g) (by omega) + obtain ⟨er, ec⟩ := rc_eq hc' (show p.ℓ * r' + c' = 4 * g + (p.ℓ * r' + c' - 4 * g) by omega) + rw [← er, ← ec, e₅, show 4 * g + (p.ℓ * r' + c' - 4 * g) = p.ℓ * r' + c' by omega, ← pA_eq] at hb + exact ⟨b, hb⟩ · cases hq' : q · obtain ⟨r', hr', c', hc', hd, hn⟩ := hbad hq' - refine ⟨r', hr', c', hc', ?_, hn⟩ - unfold Done at hd ⊢; omega + exact ⟨r', hr', c', hc', by unfold Done at hd ⊢; omega, hn⟩ · rw [hq'] at hq have h0 : rr = 0 := by rcases hrr with h1 | h0 · rw [h1] at hq; cases hq · exact h0 obtain ⟨k, hk, hn⟩ := hs0 h0 - exact ⟨r, hr, c₀ + k, by omega, by unfold Done; omega, hn⟩ - -theorem aRow_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) - {h : List (Vector Bool n)} {r : Nat} (hr : r < p.k) {s : State} (hs : S3 p h r 0 σ s) : - WP isa (aRow P p r) s (S3 p h (r + 1) 0 σ) := by - have hkl := kl_le p hp - have hg := gChk_all p hp r hr - have hl : p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by - have : ∀ p ∈ params, p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7 := by decide - exact this p hp - unfold aRow - refine WP.seq (WP.mono (aGrp_ok C hp hv hr hg.1 hs) fun s₁ h₁ => ?_) - rw [Nat.zero_add] at h₁ - by_cases h7 : p.ℓ = 7 - · rw [ite_eq_left h7] - refine WP.mono (aGrp_ok C hp hv hr (hg.2 h7) h₁) fun s₂ h₂ => ?_ - rw [show 3 + 4 = p.ℓ by omega] at h₂ - exact h₂.next - · rw [ite_eq_right h7] - refine WP.mono (seqR_ok (I := fun e => S3 p h r (e - 8 * r) σ) (p.ℓ - 4) (8 * r + 4) - (fun e he he' st hst => ?_) s₁ (by rw [show 8 * r + 4 - 8 * r = 4 by omega]; exact h₁)) fun st hst => ?_ - · have := aOne_ok C hp hv hr (c := e - 8 * r) (by omega) hst - rw [show 8 * r + (e - 8 * r) = e by omega, show e - 8 * r + 1 = e + 1 - 8 * r by omega] at this - exact this - · rw [show 8 * r + 4 + (p.ℓ - 4) - 8 * r = p.ℓ by omega] at hst - exact hst.next + obtain ⟨hr, hc⟩ := entry_lt hp (show 4 * g + k < p.k * p.ℓ by omega) + exact ⟨_, hr, _, hc, by unfold Done; have := Nat.div_add_mod (4 * g + k) p.ℓ; omega, hn⟩ /-- After the samplers, with the hint `h`. -/ structure S4 (p : Params) (h : List (Vector Bool n)) (σ st : State) : Prop where t : T p σ st hint : HintIs st.mem (pa st (pH 0)) p.k h z : ∀ i < p.ℓ, PolyIs st.mem (pa st (pZ i)) (toRq (vZ p (vSig p σ) i)) - red : ∀ r < p.k, ∀ c < p.ℓ, Reduced st.mem (pa st (pA r c)) + red : ∀ r < p.k, ∀ c < p.ℓ, Reduced st.mem (pa st (pA p.ℓ r c)) redC : Reduced st.mem (pa st pC) ok : ∃ q : Bool, st.gpr .r15 = flag (q = true) ∧ (q = true → (∀ r < p.k, ∀ c < p.ℓ, - ∃ b : Bounds, rejNTTPoly b.rejNTT (aSeed (vPk p σ) r c) = some (polyAt st.mem (pa st (pA r c)))) ∧ + ∃ b : Bounds, rejNTTPoly b.rejNTT (aSeed (vPk p σ) r c) = some (polyAt st.mem (pa st (pA p.ℓ r c)))) ∧ ∃ b : Bounds, (sampleInBall p.τ b.ball (vCt p (vSig p σ))).map toRq = some (polyAt st.mem (pa st pC))) ∧ (q = false → (∃ r < p.k, ∃ c < p.ℓ, rejNTTPoly minBounds.rejNTT (aSeed (vPk p σ) r c) = none) ∨ (sampleInBall p.τ minBounds.ball (vCt p (vSig p σ))).map toRq = none) @@ -403,7 +376,7 @@ def sChk (p : Params) : Bool := decide (32 ≤ p.pkLen) && ballChk (vB p) (vW p) (.r13, 0) p.ctildeLen pC && decide ((p.ctildeLen, p.τ) ∈ ballParams) && decide (p.ctildeLen ≤ p.sigLen) && keepChk p wsC && inB (vB p) pC 1024 && inB (vW p) pC 1024 && - (List.range p.k).all (fun r => (List.range p.ℓ).all fun c => keepB (vB p) wsC (pA r c) 1024) + (List.range p.k).all (fun r => (List.range p.ℓ).all fun c => keepB (vB p) wsC (pA p.ℓ r c) 1024) theorem sChk_all : ∀ p ∈ params, sChk p = true := by decide @@ -459,14 +432,14 @@ theorem copyK_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) · rw [hP₁.pa (show Reg.rbx ∈ bases by decide), hcp, hs.t.pkSlice hpk, List.drop_zero] rfl -theorem RhoS.s3 {p : Params} {h : List (Vector Bool n)} {σ s : State} (r : RhoS p h 4 σ s) : S3 p h 0 0 σ s := +theorem RhoS.s3 {p : Params} {h : List (Vector Bool n)} {σ s : State} (r : RhoS p h 4 σ s) : S3 p h 0 σ s := ⟨r.t, r.hint, r.z, r.rho, r.rho4, fun r' _ c' _ hd => absurd hd (by unfold Done; omega), true, by rw [r.f15]; exact flag_congr (by simp), fun _ r' _ c' _ hd => absurd hd (by unfold Done; omega), fun hq => absurd hq (by simp)⟩ theorem rhos_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) {h : List (Vector Bool n)} {s : State} (hs : S2 p h p.ℓ σ s) (h15 : s.gpr .r15 = flag True) : - WP isa rhos s (S3 p h 0 0 σ) := by + WP isa rhos s (S3 p h 0 σ) := by unfold rhos refine WP.seq (WP.mono (copyRho_ok hp hv hs h15) fun s₀ r₀ => ?_) refine WP.seq (WP.mono (copyK_ok hp hv (j := 0) (by decide) r₀) fun s₁ r₁ => ?_) @@ -475,7 +448,7 @@ theorem rhos_ok {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) exact WP.mono (copyK_ok hp hv (j := 3) (by decide) r₃) fun _ r₄ => r₄.s3 theorem ballStage_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : State} (hv : VPre p σ) - {h : List (Vector Bool n)} {s₂ : State} (hs₂ : S3 p h p.k 0 σ s₂) : + {h : List (Vector Bool n)} {s₂ : State} (hs₂ : S3 p h (p.k * p.ℓ) σ s₂) : WP isa (sampled (ballAt P (.r13, 0) p.ctildeLen p.τ pC) pC) s₂ (S4 p h σ) := by have hc := sChk_all p hp simp only [sChk, Bool.and_eq_true, decide_eq_true_eq, List.all_eq_true, List.mem_range] at hc @@ -489,14 +462,14 @@ theorem ballStage_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params have hct : bytesAt s₂.mem (pa s₂ (.r13, 0)) p.ctildeLen = vCt p (vSig p σ) := by rw [hs₂.t.sigSlice (by omega), List.drop_zero]; rfl rw [hct] at hs1 hs0 - have hdone : ∀ r' c', r' < p.k → Done p.k 0 r' c' := fun r' c' hr => by unfold Done; omega + have hdone : ∀ r' c', r' < p.k → c' < p.ℓ → Done p.ℓ (p.k * p.ℓ) r' c' := fun _ _ hr hc => done_all hr hc have ec : pa s₃ pC = pa s₂ pC := hP₃.pa (show Reg.rbx ∈ bases by decide) refine ⟨hs₂.t.step hp hv hP₃ t8, L₂.keepHint hP₃ h8 hs₂.hint, fun i hi => L₂.keepPoly hP₃ (z8 i hi) (hs₂.z i hi), - fun r hr c hc => L₂.keepRed hP₃ (c11 r hr c hc) (hs₂.red r hr c hc (hdone r c hr)), by rw [ec]; exact hred, + fun r hr c hc => L₂.keepRed hP₃ (c11 r hr c hc) (hs₂.red r hr c hc (hdone r c hr hc)), by rw [ec]; exact hred, q && (rr == 1), by rw [h15₃]; exact flag_congr (by cases q <;> simp), fun hq => ?_, fun hq => ?_⟩ · simp only [Bool.and_eq_true, beq_iff_eq] at hq refine ⟨fun r hr c hc => ?_, ?_⟩ - · obtain ⟨b, hb⟩ := hok hq.1 r hr c hc (hdone r c hr) + · obtain ⟨b, hb⟩ := hok hq.1 r hr c hc (hdone r c hr hc) exact ⟨b, by rw [hb, L₂.keepPolyAt hP₃ (c11 r hr c hc)]⟩ · obtain ⟨b, hb⟩ := hs1 hq.2 exact ⟨b, by rw [hb, ec]⟩ @@ -515,9 +488,12 @@ theorem samples_ok {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) WP isa (samples P p) s (S4 p h σ) := by unfold samples refine WP.seq (WP.mono (rhos_ok hp hv hs h15) fun s₁ s₁3 => ?_) - refine WP.seq (WP.mono (seqR_ok (I := fun r => S3 p h r 0 σ) p.k 0 (fun r _ hr st hst => aRow_ok C hp hv - (by omega) hst) s₁ s₁3) fun s₂ hs₂ => ?_) + refine WP.seq (WP.mono (seqR_ok (I := fun g => S3 p h (4 * g) σ) (p.k * p.ℓ / 4) 0 + (fun g _ hg st hst => aGrp_ok C hp hv (gChk_all p hp g (by omega)) hst) s₁ s₁3) fun s₂ hs₂ => ?_) rw [Nat.zero_add] at hs₂ - exact ballStage_ok C hp hv hs₂ + refine WP.seq (WP.mono (seqR_ok (I := fun e => S3 p h e σ) (p.k * p.ℓ % 4) (4 * (p.k * p.ℓ / 4)) + (fun e _ he st hst => aOne_ok C hp hv (by omega) hst) s₂ hs₂) fun s₃ hs₃ => ?_) + rw [Nat.div_add_mod] at hs₃ + exact ballStage_ok C hp hv hs₃ end VG.Proof.MlDsa.X86_64.Verify diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageC.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageC.lean index 7e772e828..4e05530a6 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageC.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/StageC.lean @@ -24,7 +24,7 @@ structure SC (p : Params) (h : List (Vector Bool n)) (A' : Nat → Nat → Poly) (j : Nat) (cH : Poly) (r : Nat) (σ st : State) : Prop where t : T p σ st hint : HintIs st.mem (pa st (pH 0)) p.k h - a : ∀ r' < p.k, ∀ c < p.ℓ, PolyIs st.mem (pa st (pA r' c)) (A' r' c) + a : ∀ r' < p.k, ∀ c < p.ℓ, PolyIs st.mem (pa st (pA p.ℓ r' c)) (A' r' c) z : ∀ i < p.ℓ, PolyIs st.mem (pa st (pZ i)) (if i < j then zHat p (vSig p σ) i else toRq (vZ p (vSig p σ) i)) c : PolyIs st.mem (pa st pC) cH rows : ∀ r' < r, bytesAt st.mem (pa st (sc (oB + w1Len p * r'))) (w1Len p) = @@ -36,12 +36,12 @@ but `z[ex]`, but for `C` and the rows. -/ def keepC (p : Params) (ws : List (Ptr × Nat)) (ex : Nat) : Bool := tChk p ws && keepB (vB p) ws (pH 0) (1024 * p.k) && (List.range p.ℓ).all (fun i => i == ex || keepB (vB p) ws (pZ i) 1024) && - (List.range p.k).all (fun r => (List.range p.ℓ).all fun c => keepB (vB p) ws (pA r c) 1024) + (List.range p.k).all (fun r => (List.range p.ℓ).all fun c => keepB (vB p) ws (pA p.ℓ r c) 1024) theorem keepC_spec {p : Params} {ws : List (Ptr × Nat)} {ex : Nat} (h : keepC p ws ex = true) : tChk p ws = true ∧ keepB (vB p) ws (pH 0) (1024 * p.k) = true ∧ (∀ i < p.ℓ, i ≠ ex → keepB (vB p) ws (pZ i) 1024 = true) ∧ - ∀ r < p.k, ∀ c < p.ℓ, keepB (vB p) ws (pA r c) 1024 = true := by + ∀ r < p.k, ∀ c < p.ℓ, keepB (vB p) ws (pA p.ℓ r c) 1024 = true := by simp only [keepC, Bool.and_eq_true, List.all_eq_true, List.mem_range, Bool.or_eq_true, beq_iff_eq] at h exact ⟨h.1.1.1, h.1.1.2, fun i hi hne => (h.1.2 i hi).resolve_left hne, h.2⟩ @@ -105,7 +105,7 @@ abbrev wsR (p : Params) (r : Nat) : List (Ptr × Nat) := /-- The facts about the parameters row `r` needs. -/ def rowChk (p : Params) (r : Nat) : Bool := - (List.range p.ℓ).all (fun c => mulChk (vB p) (vW p) pW (pA r c) (pZ c)) && + (List.range p.ℓ).all (fun c => mulChk (vB p) (vW p) pW (pA p.ℓ r c) (pZ c)) && t1Chk (vB p) (vW p) (.rbp, 32 + 320 * r) pT && decide (32 + 320 * r + 320 ≤ p.pkLen) && ipChk (vB p) (vW p) pT && ipChk (vB p) (vW p) pW && mulChk (vB p) (vW p) pT2 pC pT && subChk (vB p) (vW p) pW pT2 && uhChk (vB p) (vW p) (pH r) pW pW1 && decide (p.γ₂ ∈ gamma2s) && @@ -128,7 +128,7 @@ theorem wsR_bases (p : Params) (r : Nat) : ∀ w ∈ wsR p r, w.1.1 ∈ bases := /-- `rowChk`, piece by piece. -/ structure RowC (p : Params) (r : Nat) : Prop where - mul : ∀ c < p.ℓ, mulChk (vB p) (vW p) pW (pA r c) (pZ c) = true + mul : ∀ c < p.ℓ, mulChk (vB p) (vW p) pW (pA p.ℓ r c) (pZ c) = true t1 : t1Chk (vB p) (vW p) (.rbp, 32 + 320 * r) pT = true pk : 32 + 320 * r + 320 ≤ p.pkLen ipT : ipChk (vB p) (vW p) pT = true @@ -189,7 +189,7 @@ variable {P : Prims} (C : PrimsOk P) {p : Params} (hp : p ∈ params) {σ : Stat {r : Nat} (hr : r < p.k) {s₀ : State} (hs : SC p h A' Q p.ℓ cH r σ s₀) include C hp hv hr hs -theorem dotFirst_ok : WP isa (mulAt P pW (pA r 0) (pZ 0)) s₀ (DI p σ A' r 1 s₀) := by +theorem dotFirst_ok : WP isa (mulAt P pW (pA p.ℓ r 0) (pZ 0)) s₀ (DI p σ A' r 1 s₀) := by have R := rowC hp hr refine WP.mono (mulAt_ok C.mul (hs.t.lay hp hv) (R.mul 0 R.l1) (hs.a r hr 0 R.l1).1 (hs.zHat R.l1).1) fun s₁ ⟨hP₁, e₁, hq₁⟩ => ⟨hP₁, e₁, ?_⟩ @@ -198,7 +198,7 @@ theorem dotFirst_ok : WP isa (mulAt P pW (pA r 0) (pZ 0)) s₀ (DI p σ A' r 1 s exact hq₁ theorem dotStep_ok {k : Nat} (hk' : k < p.ℓ) {st : State} (hd : DI p σ A' r k s₀ st) : - WP isa (mulAddAt P pW (pA r k) (pZ k)) st (DI p σ A' r (k + 1) s₀) := by + WP isa (mulAddAt P pW (pA p.ℓ r k) (pZ k)) st (DI p σ A' r (k + 1) s₀) := by have R := rowC hp hr have L := hs.t.lay hp hv obtain ⟨_, _, hZ, hA⟩ := keepC_spec R.keep diff --git a/src/asm/x86_64/mldsa.rs b/src/asm/x86_64/mldsa.rs index 1f72903c5..cfeb41fa2 100644 --- a/src/asm/x86_64/mldsa.rs +++ b/src/asm/x86_64/mldsa.rs @@ -10770,6 +10770,4104 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_expand_mask_poly(seed: *const [u8; ) } +/// Four polynomials of `ExpandMask` (FIPS 204 Algorithm 34, lines 4 and 5): for each `k` < 4, writes `BitUnpack(H(seed, 32c), gamma1 - 1, gamma1)`, for the 66 bytes `seed` of `*seeds` from byte `66 k` and `c = 1 + bitlen (gamma1 - 1)`, to the 256 coefficients of `*a` from coefficient `256 k` (each modulo `q` = 8380417). The four are independent, so an implementation may compute them together (e.g. four SHAKE256 instances at once in vector registers). +/// +/// Contract: `VG.Spec.MlDsa.expandMask4Contract`. Constant time: only the pointers and `gamma1` may affect timing, not the data. +/// +/// It calls `vg_mldsa_expand_mask_poly` on each seed, and saves its caller's callee-saved registers in `scratch`. +/// +/// # Safety +/// +/// * `seeds` must be valid for reads of 264 bytes. +/// * `a` must be valid for reads and writes of 4096 bytes. +/// * `scratch` must be valid for reads and writes of 8192 bytes. +/// * `gamma1` must be 2^17 or 2^19. +/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `a` and `scratch` must not overlap each other or `seeds` (distinct Rust objects never do). +/// * None of `seeds`, `a` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_expand_mask_poly4(seeds: *const [u8; 264], gamma1: u32, a: *mut [u32; 1024], scratch: *mut [u64; 1024]) { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+5088], rbx", + "mov QWORD PTR [rcx+5096], rbp", + "mov QWORD PTR [rcx+5104], r12", + "mov QWORD PTR [rcx+5112], r13", + "mov QWORD PTR [rcx+5120], r14", + "mov rbx, rcx", + "mov r12, rdi", + "mov r13, rdx", + "mov r14, rsi", + "mov rdi, r12", + "add rdi, 0", + "mov rsi, r14", + "mov rdx, r13", + "add rdx, 0", + "mov rcx, rbx", + "add rcx, 6144", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, r12", + "add rdi, 66", + "mov rsi, r14", + "mov rdx, r13", + "add rdx, 1024", + "mov rcx, rbx", + "add rcx, 6144", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, r12", + "add rdi, 132", + "mov rsi, r14", + "mov rdx, r13", + "add rdx, 2048", + "mov rcx, rbx", + "add rcx, 6144", + "call {vg_mldsa_expand_mask_poly}", + "mov rdi, r12", + "add rdi, 198", + "mov rsi, r14", + "mov rdx, r13", + "add rdx, 3072", + "mov rcx, rbx", + "add rcx, 6144", + "call {vg_mldsa_expand_mask_poly}", + "mov r14, QWORD PTR [rbx+5120]", + "mov r13, QWORD PTR [rbx+5112]", + "mov r12, QWORD PTR [rbx+5104]", + "mov rbp, QWORD PTR [rbx+5096]", + "mov rbx, QWORD PTR [rbx+5088]", + "ret", + vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, + ) +} + +/// The CPU features `vg_mldsa_expand_mask_poly4_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_EXPAND_MASK_POLY4_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// Four polynomials of `ExpandMask` (FIPS 204 Algorithm 34, lines 4 and 5): for each `k` < 4, writes `BitUnpack(H(seed, 32c), gamma1 - 1, gamma1)`, for the 66 bytes `seed` of `*seeds` from byte `66 k` and `c = 1 + bitlen (gamma1 - 1)`, to the 256 coefficients of `*a` from coefficient `256 k` (each modulo `q` = 8380417). The four are independent, so an implementation may compute them together (e.g. four SHAKE256 instances at once in vector registers). +/// +/// Contract: `VG.Spec.MlDsa.expandMask4Contract`. Constant time: only the pointers and `gamma1` may affect timing, not the data. +/// +/// It runs the four instances of SHAKE256 at once, in the four 64-bit elements of AVX2 registers (as `vg_mldsa_rej_ntt_poly4_avx2` does with SHAKE128), squeezing five blocks of each, and unpacks the 576 or 640 bytes of each seed's output as `vg_mldsa_expand_mask_poly` does. It saves its caller's callee-saved registers in `scratch`. +/// +/// # Safety +/// +/// * `seeds` must be valid for reads of 264 bytes. +/// * `a` must be valid for reads and writes of 4096 bytes. +/// * `scratch` must be valid for reads and writes of 8192 bytes. +/// * `gamma1` must be 2^17 or 2^19. +/// * `scratch` is working space: on return it may hold intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `a` and `scratch` must not overlap each other or `seeds` (distinct Rust objects never do). +/// * None of `seeds`, `a` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_expand_mask_poly4_avx2(seeds: *const [u8; 264], gamma1: u32, a: *mut [u32; 1024], scratch: *mut [u64; 1024]) { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+5088], rbx", + "mov QWORD PTR [rcx+5096], rbp", + "mov QWORD PTR [rcx+5104], r12", + "mov QWORD PTR [rcx+5112], r13", + "mov QWORD PTR [rcx+5120], r14", + "mov rbx, rcx", + "mov r12, rdi", + "mov r13, rdx", + "mov r14, rsi", + "movabs rax, 1", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1600], ymm0", + "movabs rax, 32898", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1632], ymm0", + "movabs rax, -9223372036854742902", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1664], ymm0", + "movabs rax, -9223372034707259392", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1696], ymm0", + "movabs rax, 32907", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1728], ymm0", + "movabs rax, 2147483649", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1760], ymm0", + "movabs rax, -9223372034707259263", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1792], ymm0", + "movabs rax, -9223372036854743031", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1824], ymm0", + "movabs rax, 138", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1856], ymm0", + "movabs rax, 136", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1888], ymm0", + "movabs rax, 2147516425", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1920], ymm0", + "movabs rax, 2147483658", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1952], ymm0", + "movabs rax, 2147516555", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+1984], ymm0", + "movabs rax, -9223372036854775669", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2016], ymm0", + "movabs rax, -9223372036854742903", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2048], ymm0", + "movabs rax, -9223372036854743037", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2080], ymm0", + "movabs rax, -9223372036854743038", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2112], ymm0", + "movabs rax, -9223372036854775680", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2144], ymm0", + "movabs rax, 32778", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2176], ymm0", + "movabs rax, -9223372034707292150", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2208], ymm0", + "movabs rax, -9223372034707259263", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2240], ymm0", + "movabs rax, -9223372036854742912", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2272], ymm0", + "movabs rax, 2147483649", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2304], ymm0", + "movabs rax, -9223372034707259384", + "vmovq xmm0, rax", + "vpbroadcastq ymm0, xmm0", + "vmovdqu YMMWORD PTR [rbx+2336], ymm0", + "vpxor ymm0, ymm0, ymm0", + "vmovdqu YMMWORD PTR [rbx], ymm0", + "vmovdqu YMMWORD PTR [rbx+32], ymm0", + "vmovdqu YMMWORD PTR [rbx+64], ymm0", + "vmovdqu YMMWORD PTR [rbx+96], ymm0", + "vmovdqu YMMWORD PTR [rbx+128], ymm0", + "vmovdqu YMMWORD PTR [rbx+160], ymm0", + "vmovdqu YMMWORD PTR [rbx+192], ymm0", + "vmovdqu YMMWORD PTR [rbx+224], ymm0", + "vmovdqu YMMWORD PTR [rbx+256], ymm0", + "vmovdqu YMMWORD PTR [rbx+288], ymm0", + "vmovdqu YMMWORD PTR [rbx+320], ymm0", + "vmovdqu YMMWORD PTR [rbx+352], ymm0", + "vmovdqu YMMWORD PTR [rbx+384], ymm0", + "vmovdqu YMMWORD PTR [rbx+416], ymm0", + "vmovdqu YMMWORD PTR [rbx+448], ymm0", + "vmovdqu YMMWORD PTR [rbx+480], ymm0", + "vmovdqu YMMWORD PTR [rbx+512], ymm0", + "vmovdqu YMMWORD PTR [rbx+544], ymm0", + "vmovdqu YMMWORD PTR [rbx+576], ymm0", + "vmovdqu YMMWORD PTR [rbx+608], ymm0", + "vmovdqu YMMWORD PTR [rbx+640], ymm0", + "vmovdqu YMMWORD PTR [rbx+672], ymm0", + "vmovdqu YMMWORD PTR [rbx+704], ymm0", + "vmovdqu YMMWORD PTR [rbx+736], ymm0", + "vmovdqu YMMWORD PTR [rbx+768], ymm0", + "mov rax, QWORD PTR [r12]", + "mov QWORD PTR [rbx], rax", + "mov rax, QWORD PTR [r12+8]", + "mov QWORD PTR [rbx+32], rax", + "mov rax, QWORD PTR [r12+16]", + "mov QWORD PTR [rbx+64], rax", + "mov rax, QWORD PTR [r12+24]", + "mov QWORD PTR [rbx+96], rax", + "mov rax, QWORD PTR [r12+32]", + "mov QWORD PTR [rbx+128], rax", + "mov rax, QWORD PTR [r12+40]", + "mov QWORD PTR [rbx+160], rax", + "mov rax, QWORD PTR [r12+48]", + "mov QWORD PTR [rbx+192], rax", + "mov rax, QWORD PTR [r12+56]", + "mov QWORD PTR [rbx+224], rax", + "movzx eax, BYTE PTR [r12+64]", + "mov BYTE PTR [rbx+256], al", + "movzx eax, BYTE PTR [r12+65]", + "mov BYTE PTR [rbx+257], al", + "mov rax, QWORD PTR [r12+66]", + "mov QWORD PTR [rbx+8], rax", + "mov rax, QWORD PTR [r12+74]", + "mov QWORD PTR [rbx+40], rax", + "mov rax, QWORD PTR [r12+82]", + "mov QWORD PTR [rbx+72], rax", + "mov rax, QWORD PTR [r12+90]", + "mov QWORD PTR [rbx+104], rax", + "mov rax, QWORD PTR [r12+98]", + "mov QWORD PTR [rbx+136], rax", + "mov rax, QWORD PTR [r12+106]", + "mov QWORD PTR [rbx+168], rax", + "mov rax, QWORD PTR [r12+114]", + "mov QWORD PTR [rbx+200], rax", + "mov rax, QWORD PTR [r12+122]", + "mov QWORD PTR [rbx+232], rax", + "movzx eax, BYTE PTR [r12+130]", + "mov BYTE PTR [rbx+264], al", + "movzx eax, BYTE PTR [r12+131]", + "mov BYTE PTR [rbx+265], al", + "mov rax, QWORD PTR [r12+132]", + "mov QWORD PTR [rbx+16], rax", + "mov rax, QWORD PTR [r12+140]", + "mov QWORD PTR [rbx+48], rax", + "mov rax, QWORD PTR [r12+148]", + "mov QWORD PTR [rbx+80], rax", + "mov rax, QWORD PTR [r12+156]", + "mov QWORD PTR [rbx+112], rax", + "mov rax, QWORD PTR [r12+164]", + "mov QWORD PTR [rbx+144], rax", + "mov rax, QWORD PTR [r12+172]", + "mov QWORD PTR [rbx+176], rax", + "mov rax, QWORD PTR [r12+180]", + "mov QWORD PTR [rbx+208], rax", + "mov rax, QWORD PTR [r12+188]", + "mov QWORD PTR [rbx+240], rax", + "movzx eax, BYTE PTR [r12+196]", + "mov BYTE PTR [rbx+272], al", + "movzx eax, BYTE PTR [r12+197]", + "mov BYTE PTR [rbx+273], al", + "mov rax, QWORD PTR [r12+198]", + "mov QWORD PTR [rbx+24], rax", + "mov rax, QWORD PTR [r12+206]", + "mov QWORD PTR [rbx+56], rax", + "mov rax, QWORD PTR [r12+214]", + "mov QWORD PTR [rbx+88], rax", + "mov rax, QWORD PTR [r12+222]", + "mov QWORD PTR [rbx+120], rax", + "mov rax, QWORD PTR [r12+230]", + "mov QWORD PTR [rbx+152], rax", + "mov rax, QWORD PTR [r12+238]", + "mov QWORD PTR [rbx+184], rax", + "mov rax, QWORD PTR [r12+246]", + "mov QWORD PTR [rbx+216], rax", + "mov rax, QWORD PTR [r12+254]", + "mov QWORD PTR [rbx+248], rax", + "movzx eax, BYTE PTR [r12+262]", + "mov BYTE PTR [rbx+280], al", + "movzx eax, BYTE PTR [r12+263]", + "mov BYTE PTR [rbx+281], al", + "mov eax, 31", + "mov BYTE PTR [rbx+258], al", + "mov BYTE PTR [rbx+266], al", + "mov BYTE PTR [rbx+274], al", + "mov BYTE PTR [rbx+282], al", + "mov eax, 128", + "mov BYTE PTR [rbx+519], al", + "mov BYTE PTR [rbx+527], al", + "mov BYTE PTR [rbx+535], al", + "mov BYTE PTR [rbx+543], al", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "20:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 20b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2368], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2376], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2384], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2392], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2400], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2408], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2416], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2424], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2432], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2440], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2448], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2456], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2464], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2472], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2480], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2488], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2496], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3048], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3056], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3064], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3072], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3080], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3088], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3096], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3104], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3112], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3120], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3128], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3136], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3144], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3152], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3160], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3168], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3176], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3728], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3736], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3744], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3752], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3760], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3768], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3776], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3784], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3792], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3800], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3808], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3816], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3824], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3832], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3840], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3848], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3856], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4408], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4416], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4424], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4432], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4440], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4448], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4456], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4464], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4472], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4480], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4488], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4496], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4504], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4512], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4520], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4528], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4536], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "21:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 21b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2504], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2512], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2520], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2528], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2536], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2544], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2552], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2560], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2568], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2576], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2584], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2592], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2600], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2608], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2616], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2624], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2632], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3184], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3192], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3200], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3208], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3216], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3224], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3232], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3240], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3248], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3256], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3264], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3272], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3280], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3288], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3296], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3304], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3312], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+3864], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+3872], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+3880], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+3888], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+3896], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+3904], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+3912], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+3920], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+3928], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+3936], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+3944], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+3952], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+3960], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+3968], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+3976], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+3984], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+3992], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4544], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4552], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4560], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4568], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4576], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4584], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4592], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4600], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4608], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4616], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4624], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4632], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4640], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4648], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4656], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4664], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4672], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "22:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 22b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2640], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2648], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2656], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2664], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2672], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2680], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2688], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2696], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2704], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2712], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2720], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2728], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2736], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2744], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2752], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2760], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2768], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3320], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3328], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3336], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3344], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3352], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3360], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3368], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3376], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3384], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3392], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3400], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3408], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3416], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3424], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3432], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3440], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3448], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+4000], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+4008], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+4016], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+4024], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+4032], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+4040], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+4048], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+4056], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+4064], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+4072], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+4080], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+4088], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+4096], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+4104], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+4112], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+4120], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+4128], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4680], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4688], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4696], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4704], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4712], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4720], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4728], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4736], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4744], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4752], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4760], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4768], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4776], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4784], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4792], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4800], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4808], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "23:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 23b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2776], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2784], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2792], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2800], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2808], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2816], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2824], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2832], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2840], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2848], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2856], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+2864], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+2872], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+2880], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+2888], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+2896], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+2904], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3456], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3464], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3472], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3480], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3488], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3496], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3504], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3512], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3520], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3528], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3536], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3544], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3552], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3560], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3568], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3576], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3584], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+4136], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+4144], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+4152], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+4160], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+4168], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+4176], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+4184], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+4192], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+4200], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+4208], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+4216], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+4224], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+4232], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+4240], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+4248], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+4256], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+4264], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4816], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4824], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4832], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4840], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4848], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4856], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+4864], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+4872], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+4880], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+4888], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+4896], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+4904], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+4912], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+4920], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+4928], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+4936], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+4944], rax", + "mov rdi, rbx", + "mov rsi, rbx", + "add rsi, 800", + "mov rdx, rbx", + "add rdx, 1600", + "mov rcx, rbx", + "add rcx, 2368", + "24:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vmovdqu ymm10, YMMWORD PTR [rdi+160]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+320]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+480]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+640]", + "vpxor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+32]", + "vmovdqu ymm10, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+352]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+512]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+672]", + "vpxor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+64]", + "vmovdqu ymm10, YMMWORD PTR [rdi+224]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+544]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+704]", + "vpxor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+96]", + "vmovdqu ymm10, YMMWORD PTR [rdi+256]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+416]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+736]", + "vpxor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+128]", + "vmovdqu ymm10, YMMWORD PTR [rdi+288]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+448]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+608]", + "vpxor ymm4, ymm4, ymm10", + "vmovdqu ymm10, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm10", + "vpsllq ymm10, ymm1, 1", + "vpsrlq ymm11, ymm1, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm5, ymm10, ymm4", + "vpsllq ymm10, ymm2, 1", + "vpsrlq ymm11, ymm2, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm6, ymm10, ymm0", + "vpsllq ymm10, ymm3, 1", + "vpsrlq ymm11, ymm3, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm7, ymm10, ymm1", + "vpsllq ymm10, ymm4, 1", + "vpsrlq ymm11, ymm4, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm8, ymm10, ymm2", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm11, ymm0, 63", + "vpor ymm10, ymm10, ymm11", + "vpxor ymm9, ymm10, ymm3", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpxor ymm0, ymm0, ymm5", + "vmovdqu ymm1, YMMWORD PTR [rdi+192]", + "vpxor ymm1, ymm1, ymm6", + "vpsllq ymm10, ymm1, 44", + "vpsrlq ymm1, ymm1, 20", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+384]", + "vpxor ymm2, ymm2, ymm7", + "vpsllq ymm10, ymm2, 43", + "vpsrlq ymm2, ymm2, 21", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+576]", + "vpxor ymm3, ymm3, ymm8", + "vpsllq ymm10, ymm3, 21", + "vpsrlq ymm3, ymm3, 43", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+768]", + "vpxor ymm4, ymm4, ymm9", + "vpsllq ymm10, ymm4, 14", + "vpsrlq ymm4, ymm4, 50", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu ymm11, YMMWORD PTR [rdx]", + "vpxor ymm10, ymm10, ymm11", + "vmovdqu YMMWORD PTR [rsi], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+32], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+64], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+96], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+128], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+96]", + "vpxor ymm0, ymm0, ymm8", + "vpsllq ymm10, ymm0, 28", + "vpsrlq ymm0, ymm0, 36", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+288]", + "vpxor ymm1, ymm1, ymm9", + "vpsllq ymm10, ymm1, 20", + "vpsrlq ymm1, ymm1, 44", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+320]", + "vpxor ymm2, ymm2, ymm5", + "vpsllq ymm10, ymm2, 3", + "vpsrlq ymm2, ymm2, 61", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+512]", + "vpxor ymm3, ymm3, ymm6", + "vpsllq ymm10, ymm3, 45", + "vpsrlq ymm3, ymm3, 19", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+704]", + "vpxor ymm4, ymm4, ymm7", + "vpsllq ymm10, ymm4, 61", + "vpsrlq ymm4, ymm4, 3", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+160], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+192], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+224], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+256], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+288], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+32]", + "vpxor ymm0, ymm0, ymm6", + "vpsllq ymm10, ymm0, 1", + "vpsrlq ymm0, ymm0, 63", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+224]", + "vpxor ymm1, ymm1, ymm7", + "vpsllq ymm10, ymm1, 6", + "vpsrlq ymm1, ymm1, 58", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+416]", + "vpxor ymm2, ymm2, ymm8", + "vpsllq ymm10, ymm2, 25", + "vpsrlq ymm2, ymm2, 39", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+608]", + "vpxor ymm3, ymm3, ymm9", + "vpsllq ymm10, ymm3, 8", + "vpsrlq ymm3, ymm3, 56", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+640]", + "vpxor ymm4, ymm4, ymm5", + "vpsllq ymm10, ymm4, 18", + "vpsrlq ymm4, ymm4, 46", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+320], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+352], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+384], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+416], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+448], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+128]", + "vpxor ymm0, ymm0, ymm9", + "vpsllq ymm10, ymm0, 27", + "vpsrlq ymm0, ymm0, 37", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+160]", + "vpxor ymm1, ymm1, ymm5", + "vpsllq ymm10, ymm1, 36", + "vpsrlq ymm1, ymm1, 28", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+352]", + "vpxor ymm2, ymm2, ymm6", + "vpsllq ymm10, ymm2, 10", + "vpsrlq ymm2, ymm2, 54", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+544]", + "vpxor ymm3, ymm3, ymm7", + "vpsllq ymm10, ymm3, 15", + "vpsrlq ymm3, ymm3, 49", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+736]", + "vpxor ymm4, ymm4, ymm8", + "vpsllq ymm10, ymm4, 56", + "vpsrlq ymm4, ymm4, 8", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+480], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+512], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+544], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+576], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+608], ymm10", + "vmovdqu ymm0, YMMWORD PTR [rdi+64]", + "vpxor ymm0, ymm0, ymm7", + "vpsllq ymm10, ymm0, 62", + "vpsrlq ymm0, ymm0, 2", + "vpor ymm0, ymm0, ymm10", + "vmovdqu ymm1, YMMWORD PTR [rdi+256]", + "vpxor ymm1, ymm1, ymm8", + "vpsllq ymm10, ymm1, 55", + "vpsrlq ymm1, ymm1, 9", + "vpor ymm1, ymm1, ymm10", + "vmovdqu ymm2, YMMWORD PTR [rdi+448]", + "vpxor ymm2, ymm2, ymm9", + "vpsllq ymm10, ymm2, 39", + "vpsrlq ymm2, ymm2, 25", + "vpor ymm2, ymm2, ymm10", + "vmovdqu ymm3, YMMWORD PTR [rdi+480]", + "vpxor ymm3, ymm3, ymm5", + "vpsllq ymm10, ymm3, 41", + "vpsrlq ymm3, ymm3, 23", + "vpor ymm3, ymm3, ymm10", + "vmovdqu ymm4, YMMWORD PTR [rdi+672]", + "vpxor ymm4, ymm4, ymm6", + "vpsllq ymm10, ymm4, 2", + "vpsrlq ymm4, ymm4, 62", + "vpor ymm4, ymm4, ymm10", + "vpandn ymm10, ymm1, ymm2", + "vpxor ymm10, ymm10, ymm0", + "vmovdqu YMMWORD PTR [rsi+640], ymm10", + "vpandn ymm10, ymm2, ymm3", + "vpxor ymm10, ymm10, ymm1", + "vmovdqu YMMWORD PTR [rsi+672], ymm10", + "vpandn ymm10, ymm3, ymm4", + "vpxor ymm10, ymm10, ymm2", + "vmovdqu YMMWORD PTR [rsi+704], ymm10", + "vpandn ymm10, ymm4, ymm0", + "vpxor ymm10, ymm10, ymm3", + "vmovdqu YMMWORD PTR [rsi+736], ymm10", + "vpandn ymm10, ymm0, ymm1", + "vpxor ymm10, ymm10, ymm4", + "vmovdqu YMMWORD PTR [rsi+768], ymm10", + "mov rax, rdi", + "mov rdi, rsi", + "mov rsi, rax", + "add rdx, 32", + "cmp rdx, rcx", + "jne 24b", + "mov rax, QWORD PTR [rbx]", + "mov QWORD PTR [rbx+2912], rax", + "mov rax, QWORD PTR [rbx+32]", + "mov QWORD PTR [rbx+2920], rax", + "mov rax, QWORD PTR [rbx+64]", + "mov QWORD PTR [rbx+2928], rax", + "mov rax, QWORD PTR [rbx+96]", + "mov QWORD PTR [rbx+2936], rax", + "mov rax, QWORD PTR [rbx+128]", + "mov QWORD PTR [rbx+2944], rax", + "mov rax, QWORD PTR [rbx+160]", + "mov QWORD PTR [rbx+2952], rax", + "mov rax, QWORD PTR [rbx+192]", + "mov QWORD PTR [rbx+2960], rax", + "mov rax, QWORD PTR [rbx+224]", + "mov QWORD PTR [rbx+2968], rax", + "mov rax, QWORD PTR [rbx+256]", + "mov QWORD PTR [rbx+2976], rax", + "mov rax, QWORD PTR [rbx+288]", + "mov QWORD PTR [rbx+2984], rax", + "mov rax, QWORD PTR [rbx+320]", + "mov QWORD PTR [rbx+2992], rax", + "mov rax, QWORD PTR [rbx+352]", + "mov QWORD PTR [rbx+3000], rax", + "mov rax, QWORD PTR [rbx+384]", + "mov QWORD PTR [rbx+3008], rax", + "mov rax, QWORD PTR [rbx+416]", + "mov QWORD PTR [rbx+3016], rax", + "mov rax, QWORD PTR [rbx+448]", + "mov QWORD PTR [rbx+3024], rax", + "mov rax, QWORD PTR [rbx+480]", + "mov QWORD PTR [rbx+3032], rax", + "mov rax, QWORD PTR [rbx+512]", + "mov QWORD PTR [rbx+3040], rax", + "mov rax, QWORD PTR [rbx+8]", + "mov QWORD PTR [rbx+3592], rax", + "mov rax, QWORD PTR [rbx+40]", + "mov QWORD PTR [rbx+3600], rax", + "mov rax, QWORD PTR [rbx+72]", + "mov QWORD PTR [rbx+3608], rax", + "mov rax, QWORD PTR [rbx+104]", + "mov QWORD PTR [rbx+3616], rax", + "mov rax, QWORD PTR [rbx+136]", + "mov QWORD PTR [rbx+3624], rax", + "mov rax, QWORD PTR [rbx+168]", + "mov QWORD PTR [rbx+3632], rax", + "mov rax, QWORD PTR [rbx+200]", + "mov QWORD PTR [rbx+3640], rax", + "mov rax, QWORD PTR [rbx+232]", + "mov QWORD PTR [rbx+3648], rax", + "mov rax, QWORD PTR [rbx+264]", + "mov QWORD PTR [rbx+3656], rax", + "mov rax, QWORD PTR [rbx+296]", + "mov QWORD PTR [rbx+3664], rax", + "mov rax, QWORD PTR [rbx+328]", + "mov QWORD PTR [rbx+3672], rax", + "mov rax, QWORD PTR [rbx+360]", + "mov QWORD PTR [rbx+3680], rax", + "mov rax, QWORD PTR [rbx+392]", + "mov QWORD PTR [rbx+3688], rax", + "mov rax, QWORD PTR [rbx+424]", + "mov QWORD PTR [rbx+3696], rax", + "mov rax, QWORD PTR [rbx+456]", + "mov QWORD PTR [rbx+3704], rax", + "mov rax, QWORD PTR [rbx+488]", + "mov QWORD PTR [rbx+3712], rax", + "mov rax, QWORD PTR [rbx+520]", + "mov QWORD PTR [rbx+3720], rax", + "mov rax, QWORD PTR [rbx+16]", + "mov QWORD PTR [rbx+4272], rax", + "mov rax, QWORD PTR [rbx+48]", + "mov QWORD PTR [rbx+4280], rax", + "mov rax, QWORD PTR [rbx+80]", + "mov QWORD PTR [rbx+4288], rax", + "mov rax, QWORD PTR [rbx+112]", + "mov QWORD PTR [rbx+4296], rax", + "mov rax, QWORD PTR [rbx+144]", + "mov QWORD PTR [rbx+4304], rax", + "mov rax, QWORD PTR [rbx+176]", + "mov QWORD PTR [rbx+4312], rax", + "mov rax, QWORD PTR [rbx+208]", + "mov QWORD PTR [rbx+4320], rax", + "mov rax, QWORD PTR [rbx+240]", + "mov QWORD PTR [rbx+4328], rax", + "mov rax, QWORD PTR [rbx+272]", + "mov QWORD PTR [rbx+4336], rax", + "mov rax, QWORD PTR [rbx+304]", + "mov QWORD PTR [rbx+4344], rax", + "mov rax, QWORD PTR [rbx+336]", + "mov QWORD PTR [rbx+4352], rax", + "mov rax, QWORD PTR [rbx+368]", + "mov QWORD PTR [rbx+4360], rax", + "mov rax, QWORD PTR [rbx+400]", + "mov QWORD PTR [rbx+4368], rax", + "mov rax, QWORD PTR [rbx+432]", + "mov QWORD PTR [rbx+4376], rax", + "mov rax, QWORD PTR [rbx+464]", + "mov QWORD PTR [rbx+4384], rax", + "mov rax, QWORD PTR [rbx+496]", + "mov QWORD PTR [rbx+4392], rax", + "mov rax, QWORD PTR [rbx+528]", + "mov QWORD PTR [rbx+4400], rax", + "mov rax, QWORD PTR [rbx+24]", + "mov QWORD PTR [rbx+4952], rax", + "mov rax, QWORD PTR [rbx+56]", + "mov QWORD PTR [rbx+4960], rax", + "mov rax, QWORD PTR [rbx+88]", + "mov QWORD PTR [rbx+4968], rax", + "mov rax, QWORD PTR [rbx+120]", + "mov QWORD PTR [rbx+4976], rax", + "mov rax, QWORD PTR [rbx+152]", + "mov QWORD PTR [rbx+4984], rax", + "mov rax, QWORD PTR [rbx+184]", + "mov QWORD PTR [rbx+4992], rax", + "mov rax, QWORD PTR [rbx+216]", + "mov QWORD PTR [rbx+5000], rax", + "mov rax, QWORD PTR [rbx+248]", + "mov QWORD PTR [rbx+5008], rax", + "mov rax, QWORD PTR [rbx+280]", + "mov QWORD PTR [rbx+5016], rax", + "mov rax, QWORD PTR [rbx+312]", + "mov QWORD PTR [rbx+5024], rax", + "mov rax, QWORD PTR [rbx+344]", + "mov QWORD PTR [rbx+5032], rax", + "mov rax, QWORD PTR [rbx+376]", + "mov QWORD PTR [rbx+5040], rax", + "mov rax, QWORD PTR [rbx+408]", + "mov QWORD PTR [rbx+5048], rax", + "mov rax, QWORD PTR [rbx+440]", + "mov QWORD PTR [rbx+5056], rax", + "mov rax, QWORD PTR [rbx+472]", + "mov QWORD PTR [rbx+5064], rax", + "mov rax, QWORD PTR [rbx+504]", + "mov QWORD PTR [rbx+5072], rax", + "mov rax, QWORD PTR [rbx+536]", + "mov QWORD PTR [rbx+5080], rax", + "vzeroupper", + "cmp r14d, 131072", + "je 25f", + "mov rsi, rbx", + "add rsi, 2368", + "mov rdi, r13", + "add rdi, 0", + "mov ecx, 64", + "27:", + "mov eax, DWORD PTR [rsi]", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi], edx", + "mov eax, DWORD PTR [rsi+2]", + "shr eax, 4", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+4], edx", + "mov eax, DWORD PTR [rsi+5]", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+8], edx", + "mov eax, DWORD PTR [rsi+7]", + "shr eax, 4", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+12], edx", + "add rsi, 10", + "add rdi, 16", + "sub rcx, 1", + "jne 27b", + "mov rsi, rbx", + "add rsi, 3048", + "mov rdi, r13", + "add rdi, 1024", + "mov ecx, 64", + "28:", + "mov eax, DWORD PTR [rsi]", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi], edx", + "mov eax, DWORD PTR [rsi+2]", + "shr eax, 4", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+4], edx", + "mov eax, DWORD PTR [rsi+5]", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+8], edx", + "mov eax, DWORD PTR [rsi+7]", + "shr eax, 4", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+12], edx", + "add rsi, 10", + "add rdi, 16", + "sub rcx, 1", + "jne 28b", + "mov rsi, rbx", + "add rsi, 3728", + "mov rdi, r13", + "add rdi, 2048", + "mov ecx, 64", + "29:", + "mov eax, DWORD PTR [rsi]", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi], edx", + "mov eax, DWORD PTR [rsi+2]", + "shr eax, 4", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+4], edx", + "mov eax, DWORD PTR [rsi+5]", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+8], edx", + "mov eax, DWORD PTR [rsi+7]", + "shr eax, 4", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+12], edx", + "add rsi, 10", + "add rdi, 16", + "sub rcx, 1", + "jne 29b", + "mov rsi, rbx", + "add rsi, 4408", + "mov rdi, r13", + "add rdi, 3072", + "mov ecx, 64", + "210:", + "mov eax, DWORD PTR [rsi]", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi], edx", + "mov eax, DWORD PTR [rsi+2]", + "shr eax, 4", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+4], edx", + "mov eax, DWORD PTR [rsi+5]", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+8], edx", + "mov eax, DWORD PTR [rsi+7]", + "shr eax, 4", + "and eax, 1048575", + "mov edx, 524288", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+12], edx", + "add rsi, 10", + "add rdi, 16", + "sub rcx, 1", + "jne 210b", + "jmp 26f", + "25:", + "mov rsi, rbx", + "add rsi, 2368", + "mov rdi, r13", + "add rdi, 0", + "mov ecx, 64", + "211:", + "mov eax, DWORD PTR [rsi]", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi], edx", + "mov eax, DWORD PTR [rsi+2]", + "shr eax, 2", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+4], edx", + "mov eax, DWORD PTR [rsi+4]", + "shr eax, 4", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+8], edx", + "mov eax, DWORD PTR [rsi+6]", + "shr eax, 6", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+12], edx", + "add rsi, 9", + "add rdi, 16", + "sub rcx, 1", + "jne 211b", + "mov rsi, rbx", + "add rsi, 3048", + "mov rdi, r13", + "add rdi, 1024", + "mov ecx, 64", + "212:", + "mov eax, DWORD PTR [rsi]", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi], edx", + "mov eax, DWORD PTR [rsi+2]", + "shr eax, 2", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+4], edx", + "mov eax, DWORD PTR [rsi+4]", + "shr eax, 4", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+8], edx", + "mov eax, DWORD PTR [rsi+6]", + "shr eax, 6", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+12], edx", + "add rsi, 9", + "add rdi, 16", + "sub rcx, 1", + "jne 212b", + "mov rsi, rbx", + "add rsi, 3728", + "mov rdi, r13", + "add rdi, 2048", + "mov ecx, 64", + "213:", + "mov eax, DWORD PTR [rsi]", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi], edx", + "mov eax, DWORD PTR [rsi+2]", + "shr eax, 2", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+4], edx", + "mov eax, DWORD PTR [rsi+4]", + "shr eax, 4", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+8], edx", + "mov eax, DWORD PTR [rsi+6]", + "shr eax, 6", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+12], edx", + "add rsi, 9", + "add rdi, 16", + "sub rcx, 1", + "jne 213b", + "mov rsi, rbx", + "add rsi, 4408", + "mov rdi, r13", + "add rdi, 3072", + "mov ecx, 64", + "214:", + "mov eax, DWORD PTR [rsi]", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi], edx", + "mov eax, DWORD PTR [rsi+2]", + "shr eax, 2", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+4], edx", + "mov eax, DWORD PTR [rsi+4]", + "shr eax, 4", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+8], edx", + "mov eax, DWORD PTR [rsi+6]", + "shr eax, 6", + "and eax, 262143", + "mov edx, 131072", + "sub edx, eax", + "sbb eax, eax", + "and eax, 8380417", + "add edx, eax", + "mov DWORD PTR [rdi+12], edx", + "add rsi, 9", + "add rdi, 16", + "sub rcx, 1", + "jne 214b", + "26:", + "mov r14, QWORD PTR [rbx+5120]", + "mov r13, QWORD PTR [rbx+5112]", + "mov r12, QWORD PTR [rbx+5104]", + "mov rbp, QWORD PTR [rbx+5096]", + "mov rbx, QWORD PTR [rbx+5088]", + "ret", + ) +} + /// `SampleInBall` (FIPS 204 Algorithm 29): writes the polynomial with `tau` coefficients 1 or -1 and the others 0 sampled from the SHAKE256 output of the `len` bytes at `ctilde` to `*c` (each coefficient modulo `q` = 8380417), and returns 1. Returns 0 if the loop reaches its bound, which is at least 221 bytes of SHAKE256 output (FIPS 204 Appendix C; this happens with probability about 2^-256 or less): `*c` is then unspecified, and the caller must destroy it and treat the operation as failed. /// /// Contract: `VG.Spec.MlDsa.sampleInBallContract`. Not constant time in the seed: timing may depend on the pointers, `len`, `tau` and the seed `c̃` at `ctilde`, but not on anything else. diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs index ec5b8b184..a9c18e82c 100644 --- a/src/asm/x86_64/mldsa44.rs +++ b/src/asm/x86_64/mldsa44.rs @@ -1380,121 +1380,145 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov eax, 814", "mov QWORD PTR [rbx+888], rax", "27:", + "mov rdi, rbx", + "add rdi, 1296", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "28:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 28b", "mov rax, QWORD PTR [rbx+896]", "add rax, 0", - "mov BYTE PTR [rbx+1024], al", + "mov BYTE PTR [rbx+1360], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1361], al", + "mov rdi, rbx", + "add rdi, 1362", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "29:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 29b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1426], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1427], al", + "mov rdi, rbx", + "add rdi, 1428", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "210:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 210b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1492], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1493], al", + "mov rdi, rbx", + "add rdi, 1494", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "211:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 211b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1558], al", "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "mov BYTE PTR [rbx+1559], al", "mov rdi, rbx", - "add rdi, 960", + "add rdi, 1296", "mov esi, 131072", "mov rdx, rbx", "add rdx, 14336", "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rcx, 55296", + "call {vg_mldsa_expand_mask_poly4_avx2}", "mov rdi, rbx", "add rdi, 18432", "mov rsi, rbx", "add rsi, 14336", "mov ecx, 128", - "28:", + "212:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 28b", + "jne 212b", "mov rdi, rbx", "add rdi, 18432", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 1", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 131072", - "mov rdx, rbx", - "add rdx, 15360", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 19456", "mov rsi, rbx", "add rsi, 15360", "mov ecx, 128", - "29:", + "213:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 29b", + "jne 213b", "mov rdi, rbx", "add rdi, 19456", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 2", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 131072", - "mov rdx, rbx", - "add rdx, 16384", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 20480", "mov rsi, rbx", "add rsi, 16384", "mov ecx, 128", - "210:", + "214:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 210b", + "jne 214b", "mov rdi, rbx", "add rdi, 20480", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 3", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 131072", - "mov rdx, rbx", - "add rdx, 17408", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 21504", "mov rsi, rbx", "add rsi, 17408", "mov ecx, 128", - "211:", + "215:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 211b", + "jne 215b", "mov rdi, rbx", "add rdi, 21504", "mov rsi, rbx", @@ -1758,12 +1782,12 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "add r8, 3072", "call {vg_mldsa_sample_in_ball}", "test eax, eax", - "jne 212f", + "jne 216f", "mov r15d, 0", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "jmp 213f", - "212:", + "jmp 217f", + "216:", "mov rdi, rbx", "add rdi, 5120", "mov rsi, rbx", @@ -1994,13 +2018,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rsi, rbx", "add rsi, 22528", "mov ecx, 128", - "214:", + "218:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 214b", + "jne 218b", "mov rdi, rbx", "add rdi, 22528", "mov rsi, rbx", @@ -2044,13 +2068,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rsi, rbx", "add rsi, 23552", "mov ecx, 128", - "215:", + "219:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 215b", + "jne 219b", "mov rdi, rbx", "add rdi, 23552", "mov rsi, rbx", @@ -2094,13 +2118,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rsi, rbx", "add rsi, 24576", "mov ecx, 128", - "216:", + "220:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 216b", + "jne 220b", "mov rdi, rbx", "add rdi, 24576", "mov rsi, rbx", @@ -2144,13 +2168,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rsi, rbx", "add rsi, 25600", "mov ecx, 128", - "217:", + "221:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 217b", + "jne 221b", "mov rdi, rbx", "add rdi, 25600", "mov rsi, rbx", @@ -2177,36 +2201,36 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "shr rax, 63", "and r15d, eax", "test r15d, r15d", - "jne 218f", + "jne 222f", "mov rax, QWORD PTR [rbx+896]", "add rax, 4", "mov QWORD PTR [rbx+896], rax", - "jmp 219f", - "218:", + "jmp 223f", + "222:", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "219:", - "213:", + "223:", + "217:", "mov rax, QWORD PTR [rbx+888]", "sub rax, 1", "mov QWORD PTR [rbx+888], rax", "jne 27b", "test r15d, r15d", - "jne 220f", - "jmp 221f", - "220:", + "jne 224f", + "jmp 225f", + "224:", "mov rdi, r14", "add rdi, 0", "mov rsi, rbx", "add rsi, 1040", "mov ecx, 4", - "222:", + "226:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 222b", + "jne 226b", "mov rdi, rbx", "add rdi, 14336", "mov esi, 131071", @@ -2247,7 +2271,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "add rcx, 2336", "mov r8d, 84", "call {vg_mldsa_hint_bit_pack}", - "221:", + "225:", "26:", "mov eax, r15d", "mov r15, QWORD PTR [rbx+880]", @@ -2263,7 +2287,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, - vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, + vg_mldsa_expand_mask_poly4_avx2 = sym super::mldsa::vg_mldsa_expand_mask_poly4_avx2, vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, @@ -2649,7 +2673,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -2657,7 +2681,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", + "add rdi, 32768", "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", @@ -2685,7 +2709,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -2693,7 +2717,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 45056", + "add rdi, 36864", "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", @@ -2721,7 +2745,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -2729,7 +2753,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", + "add rdi, 40960", "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", @@ -2858,28 +2882,28 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -2928,28 +2952,28 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -2998,28 +3022,28 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -4748,121 +4772,145 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: "mov eax, 814", "mov QWORD PTR [rbx+888], rax", "27:", + "mov rdi, rbx", + "add rdi, 1296", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "28:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 28b", "mov rax, QWORD PTR [rbx+896]", "add rax, 0", - "mov BYTE PTR [rbx+1024], al", + "mov BYTE PTR [rbx+1360], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1361], al", + "mov rdi, rbx", + "add rdi, 1362", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "29:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 29b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1426], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1427], al", + "mov rdi, rbx", + "add rdi, 1428", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "210:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 210b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1492], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1493], al", + "mov rdi, rbx", + "add rdi, 1494", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "211:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 211b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1558], al", "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "mov BYTE PTR [rbx+1559], al", "mov rdi, rbx", - "add rdi, 960", + "add rdi, 1296", "mov esi, 131072", "mov rdx, rbx", "add rdx, 14336", "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rcx, 55296", + "call {vg_mldsa_expand_mask_poly4}", "mov rdi, rbx", "add rdi, 18432", "mov rsi, rbx", "add rsi, 14336", "mov ecx, 128", - "28:", + "212:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 28b", + "jne 212b", "mov rdi, rbx", "add rdi, 18432", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 1", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 131072", - "mov rdx, rbx", - "add rdx, 15360", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 19456", "mov rsi, rbx", "add rsi, 15360", "mov ecx, 128", - "29:", + "213:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 29b", + "jne 213b", "mov rdi, rbx", "add rdi, 19456", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 2", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 131072", - "mov rdx, rbx", - "add rdx, 16384", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 20480", "mov rsi, rbx", "add rsi, 16384", "mov ecx, 128", - "210:", + "214:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 210b", + "jne 214b", "mov rdi, rbx", "add rdi, 20480", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 3", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 131072", - "mov rdx, rbx", - "add rdx, 17408", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 21504", "mov rsi, rbx", "add rsi, 17408", "mov ecx, 128", - "211:", + "215:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 211b", + "jne 215b", "mov rdi, rbx", "add rdi, 21504", "mov rsi, rbx", @@ -5126,12 +5174,12 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: "add r8, 3072", "call {vg_mldsa_sample_in_ball}", "test eax, eax", - "jne 212f", + "jne 216f", "mov r15d, 0", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "jmp 213f", - "212:", + "jmp 217f", + "216:", "mov rdi, rbx", "add rdi, 5120", "mov rsi, rbx", @@ -5362,13 +5410,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: "mov rsi, rbx", "add rsi, 22528", "mov ecx, 128", - "214:", + "218:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 214b", + "jne 218b", "mov rdi, rbx", "add rdi, 22528", "mov rsi, rbx", @@ -5412,13 +5460,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: "mov rsi, rbx", "add rsi, 23552", "mov ecx, 128", - "215:", + "219:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 215b", + "jne 219b", "mov rdi, rbx", "add rdi, 23552", "mov rsi, rbx", @@ -5462,13 +5510,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: "mov rsi, rbx", "add rsi, 24576", "mov ecx, 128", - "216:", + "220:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 216b", + "jne 220b", "mov rdi, rbx", "add rdi, 24576", "mov rsi, rbx", @@ -5512,13 +5560,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: "mov rsi, rbx", "add rsi, 25600", "mov ecx, 128", - "217:", + "221:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 217b", + "jne 221b", "mov rdi, rbx", "add rdi, 25600", "mov rsi, rbx", @@ -5545,36 +5593,36 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: "shr rax, 63", "and r15d, eax", "test r15d, r15d", - "jne 218f", + "jne 222f", "mov rax, QWORD PTR [rbx+896]", "add rax, 4", "mov QWORD PTR [rbx+896], rax", - "jmp 219f", - "218:", + "jmp 223f", + "222:", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "219:", - "213:", + "223:", + "217:", "mov rax, QWORD PTR [rbx+888]", "sub rax, 1", "mov QWORD PTR [rbx+888], rax", "jne 27b", "test r15d, r15d", - "jne 220f", - "jmp 221f", - "220:", + "jne 224f", + "jmp 225f", + "224:", "mov rdi, r14", "add rdi, 0", "mov rsi, rbx", "add rsi, 1040", "mov ecx, 4", - "222:", + "226:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 222b", + "jne 226b", "mov rdi, rbx", "add rdi, 14336", "mov esi, 131071", @@ -5615,7 +5663,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: "add rcx, 2336", "mov r8d, 84", "call {vg_mldsa_hint_bit_pack}", - "221:", + "225:", "26:", "mov eax, r15d", "mov r15, QWORD PTR [rbx+880]", @@ -5631,7 +5679,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, - vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, + vg_mldsa_expand_mask_poly4 = sym super::mldsa::vg_mldsa_expand_mask_poly4, vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, vg_mldsa_multiply_add_ntt = sym super::mldsa::vg_mldsa_multiply_add_ntt, vg_mldsa_inv_ntt = sym super::mldsa::vg_mldsa_inv_ntt, @@ -6009,7 +6057,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_rej_ntt_poly4}", @@ -6017,7 +6065,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", + "add rdi, 32768", "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", @@ -6045,7 +6093,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_rej_ntt_poly4}", @@ -6053,7 +6101,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 45056", + "add rdi, 36864", "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", @@ -6081,7 +6129,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_rej_ntt_poly4}", @@ -6089,7 +6137,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", + "add rdi, 40960", "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", @@ -6218,28 +6266,28 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", @@ -6288,28 +6336,28 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", @@ -6358,28 +6406,28 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs index 9468f6238..cf7fcce39 100644 --- a/src/asm/x86_64/mldsa65.rs +++ b/src/asm/x86_64/mldsa65.rs @@ -1958,121 +1958,145 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov eax, 814", "mov QWORD PTR [rbx+888], rax", "27:", + "mov rdi, rbx", + "add rdi, 1296", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "28:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 28b", "mov rax, QWORD PTR [rbx+896]", "add rax, 0", - "mov BYTE PTR [rbx+1024], al", + "mov BYTE PTR [rbx+1360], al", "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "mov BYTE PTR [rbx+1361], al", "mov rdi, rbx", - "add rdi, 960", + "add rdi, 1362", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "29:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 29b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1426], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1427], al", + "mov rdi, rbx", + "add rdi, 1428", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "210:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 210b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1492], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1493], al", + "mov rdi, rbx", + "add rdi, 1494", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "211:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 211b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1558], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1559], al", + "mov rdi, rbx", + "add rdi, 1296", "mov esi, 524288", "mov rdx, rbx", "add rdx, 16384", "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rcx, 80896", + "call {vg_mldsa_expand_mask_poly4_avx2}", "mov rdi, rbx", "add rdi, 21504", "mov rsi, rbx", "add rsi, 16384", "mov ecx, 128", - "28:", + "212:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 28b", + "jne 212b", "mov rdi, rbx", "add rdi, 21504", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 1", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 17408", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 22528", "mov rsi, rbx", "add rsi, 17408", "mov ecx, 128", - "29:", + "213:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 29b", + "jne 213b", "mov rdi, rbx", "add rdi, 22528", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 2", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 18432", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 23552", "mov rsi, rbx", "add rsi, 18432", "mov ecx, 128", - "210:", + "214:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 210b", + "jne 214b", "mov rdi, rbx", "add rdi, 23552", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 3", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 19456", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 24576", "mov rsi, rbx", "add rsi, 19456", "mov ecx, 128", - "211:", + "215:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 211b", + "jne 215b", "mov rdi, rbx", "add rdi, 24576", "mov rsi, rbx", @@ -2096,13 +2120,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rsi, rbx", "add rsi, 20480", "mov ecx, 128", - "212:", + "216:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 212b", + "jne 216b", "mov rdi, rbx", "add rdi, 25600", "mov rsi, rbx", @@ -2500,12 +2524,12 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "add r8, 3072", "call {vg_mldsa_sample_in_ball}", "test eax, eax", - "jne 213f", + "jne 217f", "mov r15d, 0", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "jmp 214f", - "213:", + "jmp 218f", + "217:", "mov rdi, rbx", "add rdi, 5120", "mov rsi, rbx", @@ -2814,13 +2838,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rsi, rbx", "add rsi, 26624", "mov ecx, 128", - "215:", + "219:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 215b", + "jne 219b", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", @@ -2864,13 +2888,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rsi, rbx", "add rsi, 27648", "mov ecx, 128", - "216:", + "220:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 216b", + "jne 220b", "mov rdi, rbx", "add rdi, 27648", "mov rsi, rbx", @@ -2914,13 +2938,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rsi, rbx", "add rsi, 28672", "mov ecx, 128", - "217:", + "221:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 217b", + "jne 221b", "mov rdi, rbx", "add rdi, 28672", "mov rsi, rbx", @@ -2964,13 +2988,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rsi, rbx", "add rsi, 29696", "mov ecx, 128", - "218:", + "222:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 218b", + "jne 222b", "mov rdi, rbx", "add rdi, 29696", "mov rsi, rbx", @@ -3014,13 +3038,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rsi, rbx", "add rsi, 30720", "mov ecx, 128", - "219:", + "223:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 219b", + "jne 223b", "mov rdi, rbx", "add rdi, 30720", "mov rsi, rbx", @@ -3064,13 +3088,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rsi, rbx", "add rsi, 31744", "mov ecx, 128", - "220:", + "224:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 220b", + "jne 224b", "mov rdi, rbx", "add rdi, 31744", "mov rsi, rbx", @@ -3097,36 +3121,36 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "shr rax, 63", "and r15d, eax", "test r15d, r15d", - "jne 221f", + "jne 225f", "mov rax, QWORD PTR [rbx+896]", "add rax, 5", "mov QWORD PTR [rbx+896], rax", - "jmp 222f", - "221:", + "jmp 226f", + "225:", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "222:", - "214:", + "226:", + "218:", "mov rax, QWORD PTR [rbx+888]", "sub rax, 1", "mov QWORD PTR [rbx+888], rax", "jne 27b", "test r15d, r15d", - "jne 223f", - "jmp 224f", - "223:", + "jne 227f", + "jmp 228f", + "227:", "mov rdi, r14", "add rdi, 0", "mov rsi, rbx", "add rsi, 1040", "mov ecx, 6", - "225:", + "229:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 225b", + "jne 229b", "mov rdi, rbx", "add rdi, 16384", "mov esi, 524287", @@ -3175,7 +3199,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "add rcx, 3248", "mov r8d, 61", "call {vg_mldsa_hint_bit_pack}", - "224:", + "228:", "26:", "mov eax, r15d", "mov r15, QWORD PTR [rbx+880]", @@ -3192,6 +3216,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_expand_mask_poly4_avx2 = sym super::mldsa::vg_mldsa_expand_mask_poly4_avx2, vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, @@ -3573,22 +3598,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "sub rcx, 1", "jne 29b", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", "add rsi, 32768", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", "add rdi, 32768", - "mov ecx, 256", + "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -3596,21 +3633,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 210b", - "mov eax, 0", + "mov eax, 3", "mov BYTE PTR [rbx+2592], al", "mov eax, 1", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+2626], al", "mov eax, 1", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 0", "mov BYTE PTR [rbx+2660], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", - "mov BYTE PTR [rbx+2694], al", "mov eax, 1", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", @@ -3632,23 +3669,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 211b", + "mov eax, 2", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", "add rsi, 40960", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", "add rdi, 40960", - "mov ecx, 256", + "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -3656,21 +3705,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 212b", - "mov eax, 0", + "mov eax, 1", "mov BYTE PTR [rbx+2592], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", - "mov BYTE PTR [rbx+2626], al", "mov eax, 2", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2660], al", - "mov eax, 2", - "mov BYTE PTR [rbx+2661], al", "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 4", "mov BYTE PTR [rbx+2694], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", @@ -3692,23 +3741,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 213b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", "add rsi, 49152", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", "add rdi, 49152", - "mov ecx, 256", + "mov ecx, 1024", "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -3716,21 +3777,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 214b", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+2592], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 0", "mov BYTE PTR [rbx+2626], al", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 1", "mov BYTE PTR [rbx+2660], al", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 2", "mov BYTE PTR [rbx+2694], al", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", @@ -3752,9 +3813,9 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 215b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", "mov eax, 3", + "mov BYTE PTR [rbx+928], al", + "mov eax, 5", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", @@ -3776,102 +3837,6 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 216b", - "mov eax, 0", - "mov BYTE PTR [rbx+2592], al", - "mov eax, 4", - "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", - "mov BYTE PTR [rbx+2626], al", - "mov eax, 4", - "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", - "mov BYTE PTR [rbx+2660], al", - "mov eax, 4", - "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", - "mov BYTE PTR [rbx+2694], al", - "mov eax, 4", - "mov BYTE PTR [rbx+2695], al", - "mov rdi, rbx", - "add rdi, 2560", - "mov rsi, rbx", - "add rsi, 61440", - "mov rdx, rbx", - "add rdx, 77824", - "call {vg_mldsa_rej_ntt_poly4_avx2}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 1024", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 65536", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 0", - "mov BYTE PTR [rbx+2592], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", - "mov BYTE PTR [rbx+2626], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", - "mov BYTE PTR [rbx+2660], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", - "mov BYTE PTR [rbx+2694], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2695], al", - "mov rdi, rbx", - "add rdi, 2560", - "mov rsi, rbx", - "add rsi, 69632", - "mov rdx, rbx", - "add rdx, 77824", - "call {vg_mldsa_rej_ntt_poly4_avx2}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 1024", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 5", @@ -3879,7 +3844,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -3887,15 +3852,15 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 73728", + "add rdi, 58368", "mov ecx, 256", - "220:", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 217b", "mov rdi, r13", "add rdi, 0", "mov esi, 48", @@ -3911,13 +3876,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 218b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -4028,35 +3993,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -4105,35 +4070,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -4182,35 +4147,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -4259,35 +4224,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -4336,35 +4301,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -4480,7 +4445,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 0", "mov ecx, 48", "mov edx, 0", - "222:", + "219:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -4488,7 +4453,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 222b", + "jne 219b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -6672,121 +6637,145 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "mov eax, 814", "mov QWORD PTR [rbx+888], rax", "27:", + "mov rdi, rbx", + "add rdi, 1296", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "28:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 28b", "mov rax, QWORD PTR [rbx+896]", "add rax, 0", - "mov BYTE PTR [rbx+1024], al", + "mov BYTE PTR [rbx+1360], al", "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "mov BYTE PTR [rbx+1361], al", "mov rdi, rbx", - "add rdi, 960", + "add rdi, 1362", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "29:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 29b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1426], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1427], al", + "mov rdi, rbx", + "add rdi, 1428", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "210:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 210b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1492], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1493], al", + "mov rdi, rbx", + "add rdi, 1494", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "211:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 211b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1558], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1559], al", + "mov rdi, rbx", + "add rdi, 1296", "mov esi, 524288", "mov rdx, rbx", "add rdx, 16384", "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rcx, 80896", + "call {vg_mldsa_expand_mask_poly4}", "mov rdi, rbx", "add rdi, 21504", "mov rsi, rbx", "add rsi, 16384", "mov ecx, 128", - "28:", + "212:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 28b", + "jne 212b", "mov rdi, rbx", "add rdi, 21504", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 1", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 17408", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 22528", "mov rsi, rbx", "add rsi, 17408", "mov ecx, 128", - "29:", + "213:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 29b", + "jne 213b", "mov rdi, rbx", "add rdi, 22528", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 2", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 18432", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 23552", "mov rsi, rbx", "add rsi, 18432", "mov ecx, 128", - "210:", + "214:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 210b", + "jne 214b", "mov rdi, rbx", "add rdi, 23552", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 3", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 19456", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 24576", "mov rsi, rbx", "add rsi, 19456", "mov ecx, 128", - "211:", + "215:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 211b", + "jne 215b", "mov rdi, rbx", "add rdi, 24576", "mov rsi, rbx", @@ -6810,13 +6799,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "mov rsi, rbx", "add rsi, 20480", "mov ecx, 128", - "212:", + "216:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 212b", + "jne 216b", "mov rdi, rbx", "add rdi, 25600", "mov rsi, rbx", @@ -7214,12 +7203,12 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "add r8, 3072", "call {vg_mldsa_sample_in_ball}", "test eax, eax", - "jne 213f", + "jne 217f", "mov r15d, 0", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "jmp 214f", - "213:", + "jmp 218f", + "217:", "mov rdi, rbx", "add rdi, 5120", "mov rsi, rbx", @@ -7528,13 +7517,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "mov rsi, rbx", "add rsi, 26624", "mov ecx, 128", - "215:", + "219:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 215b", + "jne 219b", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", @@ -7578,13 +7567,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "mov rsi, rbx", "add rsi, 27648", "mov ecx, 128", - "216:", + "220:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 216b", + "jne 220b", "mov rdi, rbx", "add rdi, 27648", "mov rsi, rbx", @@ -7628,13 +7617,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "mov rsi, rbx", "add rsi, 28672", "mov ecx, 128", - "217:", + "221:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 217b", + "jne 221b", "mov rdi, rbx", "add rdi, 28672", "mov rsi, rbx", @@ -7678,13 +7667,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "mov rsi, rbx", "add rsi, 29696", "mov ecx, 128", - "218:", + "222:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 218b", + "jne 222b", "mov rdi, rbx", "add rdi, 29696", "mov rsi, rbx", @@ -7728,13 +7717,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "mov rsi, rbx", "add rsi, 30720", "mov ecx, 128", - "219:", + "223:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 219b", + "jne 223b", "mov rdi, rbx", "add rdi, 30720", "mov rsi, rbx", @@ -7778,13 +7767,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "mov rsi, rbx", "add rsi, 31744", "mov ecx, 128", - "220:", + "224:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 220b", + "jne 224b", "mov rdi, rbx", "add rdi, 31744", "mov rsi, rbx", @@ -7811,36 +7800,36 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "shr rax, 63", "and r15d, eax", "test r15d, r15d", - "jne 221f", + "jne 225f", "mov rax, QWORD PTR [rbx+896]", "add rax, 5", "mov QWORD PTR [rbx+896], rax", - "jmp 222f", - "221:", + "jmp 226f", + "225:", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "222:", - "214:", + "226:", + "218:", "mov rax, QWORD PTR [rbx+888]", "sub rax, 1", "mov QWORD PTR [rbx+888], rax", "jne 27b", "test r15d, r15d", - "jne 223f", - "jmp 224f", - "223:", + "jne 227f", + "jmp 228f", + "227:", "mov rdi, r14", "add rdi, 0", "mov rsi, rbx", "add rsi, 1040", "mov ecx, 6", - "225:", + "229:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 225b", + "jne 229b", "mov rdi, rbx", "add rdi, 16384", "mov esi, 524287", @@ -7889,7 +7878,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: "add rcx, 3248", "mov r8d, 61", "call {vg_mldsa_hint_bit_pack}", - "224:", + "228:", "26:", "mov eax, r15d", "mov r15, QWORD PTR [rbx+880]", @@ -7906,6 +7895,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_expand_mask_poly4 = sym super::mldsa::vg_mldsa_expand_mask_poly4, vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, vg_mldsa_multiply_add_ntt = sym super::mldsa::vg_mldsa_multiply_add_ntt, @@ -8279,22 +8269,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "sub rcx, 1", "jne 29b", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", "add rsi, 32768", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", "add rdi, 32768", - "mov ecx, 256", + "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -8302,21 +8304,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rdi, 4", "sub rcx, 1", "jne 210b", - "mov eax, 0", + "mov eax, 3", "mov BYTE PTR [rbx+2592], al", "mov eax, 1", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+2626], al", "mov eax, 1", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 0", "mov BYTE PTR [rbx+2660], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", - "mov BYTE PTR [rbx+2694], al", "mov eax, 1", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", @@ -8338,23 +8340,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rdi, 4", "sub rcx, 1", "jne 211b", + "mov eax, 2", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", "add rsi, 40960", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", "add rdi, 40960", - "mov ecx, 256", + "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -8362,21 +8376,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rdi, 4", "sub rcx, 1", "jne 212b", - "mov eax, 0", + "mov eax, 1", "mov BYTE PTR [rbx+2592], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", - "mov BYTE PTR [rbx+2626], al", "mov eax, 2", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2660], al", - "mov eax, 2", - "mov BYTE PTR [rbx+2661], al", "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 4", "mov BYTE PTR [rbx+2694], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", @@ -8398,50 +8412,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rdi, 4", "sub rcx, 1", "jne 213b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", "mov eax, 0", "mov BYTE PTR [rbx+2592], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2593], al", "mov eax, 1", "mov BYTE PTR [rbx+2626], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2627], al", "mov eax, 2", "mov BYTE PTR [rbx+2660], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2661], al", "mov eax, 3", "mov BYTE PTR [rbx+2694], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 77824", "call {vg_mldsa_rej_ntt_poly4}", @@ -8449,59 +8439,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", + "add rdi, 49152", "mov ecx, 1024", - "215:", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 215b", + "jne 214b", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", - "mov eax, 0", "mov BYTE PTR [rbx+2592], al", "mov eax, 4", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 0", "mov BYTE PTR [rbx+2626], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 1", "mov BYTE PTR [rbx+2660], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 2", "mov BYTE PTR [rbx+2694], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 77824", "call {vg_mldsa_rej_ntt_poly4}", @@ -8509,23 +8475,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 61440", + "add rdi, 53248", "mov ecx, 1024", - "217:", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 217b", - "mov eax, 4", + "jne 215b", + "mov eax, 3", "mov BYTE PTR [rbx+928], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -8533,51 +8499,15 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 65536", + "add rdi, 57344", "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 0", - "mov BYTE PTR [rbx+2592], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", - "mov BYTE PTR [rbx+2626], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", - "mov BYTE PTR [rbx+2660], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", - "mov BYTE PTR [rbx+2694], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2695], al", - "mov rdi, rbx", - "add rdi, 2560", - "mov rsi, rbx", - "add rsi, 69632", - "mov rdx, rbx", - "add rdx, 77824", - "call {vg_mldsa_rej_ntt_poly4}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 1024", - "219:", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 219b", + "jne 216b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 5", @@ -8585,7 +8515,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -8593,15 +8523,15 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 73728", + "add rdi, 58368", "mov ecx, 256", - "220:", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 217b", "mov rdi, r13", "add rdi, 0", "mov esi, 48", @@ -8617,13 +8547,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 218b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -8734,35 +8664,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", @@ -8811,35 +8741,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", @@ -8888,35 +8818,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", @@ -8965,35 +8895,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", @@ -9042,35 +8972,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", @@ -9186,7 +9116,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rdi, 0", "mov ecx, 48", "mov edx, 0", - "222:", + "219:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -9194,7 +9124,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 222b", + "jne 219b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs index 00b263cf1..c4f9fea7f 100644 --- a/src/asm/x86_64/mldsa87.rs +++ b/src/asm/x86_64/mldsa87.rs @@ -2763,121 +2763,145 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov eax, 814", "mov QWORD PTR [rbx+888], rax", "27:", + "mov rdi, rbx", + "add rdi, 1296", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "28:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 28b", "mov rax, QWORD PTR [rbx+896]", "add rax, 0", - "mov BYTE PTR [rbx+1024], al", + "mov BYTE PTR [rbx+1360], al", "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "mov BYTE PTR [rbx+1361], al", "mov rdi, rbx", - "add rdi, 960", + "add rdi, 1362", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "29:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 29b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1426], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1427], al", + "mov rdi, rbx", + "add rdi, 1428", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "210:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 210b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1492], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1493], al", + "mov rdi, rbx", + "add rdi, 1494", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "211:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 211b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1558], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1559], al", + "mov rdi, rbx", + "add rdi, 1296", "mov esi, 524288", "mov rdx, rbx", "add rdx, 18432", "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rcx, 121856", + "call {vg_mldsa_expand_mask_poly4_avx2}", "mov rdi, rbx", "add rdi, 25600", "mov rsi, rbx", "add rsi, 18432", "mov ecx, 128", - "28:", + "212:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 28b", + "jne 212b", "mov rdi, rbx", "add rdi, 25600", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 1", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 19456", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", "add rsi, 19456", "mov ecx, 128", - "29:", + "213:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 29b", + "jne 213b", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 2", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 20480", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 27648", "mov rsi, rbx", "add rsi, 20480", "mov ecx, 128", - "210:", + "214:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 210b", + "jne 214b", "mov rdi, rbx", "add rdi, 27648", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt_avx2}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 3", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 21504", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 28672", "mov rsi, rbx", "add rsi, 21504", "mov ecx, 128", - "211:", + "215:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 211b", + "jne 215b", "mov rdi, rbx", "add rdi, 28672", "mov rsi, rbx", @@ -2901,13 +2925,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 22528", "mov ecx, 128", - "212:", + "216:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 212b", + "jne 216b", "mov rdi, rbx", "add rdi, 29696", "mov rsi, rbx", @@ -2931,13 +2955,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 23552", "mov ecx, 128", - "213:", + "217:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 213b", + "jne 217b", "mov rdi, rbx", "add rdi, 30720", "mov rsi, rbx", @@ -2961,13 +2985,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 24576", "mov ecx, 128", - "214:", + "218:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 214b", + "jne 218b", "mov rdi, rbx", "add rdi, 31744", "mov rsi, rbx", @@ -3583,12 +3607,12 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "add r8, 3072", "call {vg_mldsa_sample_in_ball}", "test eax, eax", - "jne 215f", + "jne 219f", "mov r15d, 0", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "jmp 216f", - "215:", + "jmp 220f", + "219:", "mov rdi, rbx", "add rdi, 5120", "mov rsi, rbx", @@ -3997,13 +4021,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 32768", "mov ecx, 128", - "217:", + "221:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 217b", + "jne 221b", "mov rdi, rbx", "add rdi, 32768", "mov rsi, rbx", @@ -4047,13 +4071,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 33792", "mov ecx, 128", - "218:", + "222:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 218b", + "jne 222b", "mov rdi, rbx", "add rdi, 33792", "mov rsi, rbx", @@ -4097,13 +4121,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 34816", "mov ecx, 128", - "219:", + "223:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 219b", + "jne 223b", "mov rdi, rbx", "add rdi, 34816", "mov rsi, rbx", @@ -4147,13 +4171,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 35840", "mov ecx, 128", - "220:", + "224:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 220b", + "jne 224b", "mov rdi, rbx", "add rdi, 35840", "mov rsi, rbx", @@ -4197,13 +4221,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 36864", "mov ecx, 128", - "221:", + "225:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 221b", + "jne 225b", "mov rdi, rbx", "add rdi, 36864", "mov rsi, rbx", @@ -4247,13 +4271,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 37888", "mov ecx, 128", - "222:", + "226:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 222b", + "jne 226b", "mov rdi, rbx", "add rdi, 37888", "mov rsi, rbx", @@ -4297,13 +4321,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 38912", "mov ecx, 128", - "223:", + "227:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 223b", + "jne 227b", "mov rdi, rbx", "add rdi, 38912", "mov rsi, rbx", @@ -4347,13 +4371,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rsi, rbx", "add rsi, 39936", "mov ecx, 128", - "224:", + "228:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 224b", + "jne 228b", "mov rdi, rbx", "add rdi, 39936", "mov rsi, rbx", @@ -4380,36 +4404,36 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "shr rax, 63", "and r15d, eax", "test r15d, r15d", - "jne 225f", + "jne 229f", "mov rax, QWORD PTR [rbx+896]", "add rax, 7", "mov QWORD PTR [rbx+896], rax", - "jmp 226f", - "225:", + "jmp 230f", + "229:", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "226:", - "216:", + "230:", + "220:", "mov rax, QWORD PTR [rbx+888]", "sub rax, 1", "mov QWORD PTR [rbx+888], rax", "jne 27b", "test r15d, r15d", - "jne 227f", - "jmp 228f", - "227:", + "jne 231f", + "jmp 232f", + "231:", "mov rdi, r14", "add rdi, 0", "mov rsi, rbx", "add rsi, 1040", "mov ecx, 8", - "229:", + "233:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 229b", + "jne 233b", "mov rdi, rbx", "add rdi, 18432", "mov esi, 524287", @@ -4474,7 +4498,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "add rcx, 4544", "mov r8d, 83", "call {vg_mldsa_hint_bit_pack}", - "228:", + "232:", "26:", "mov eax, r15d", "mov r15, QWORD PTR [rbx+880]", @@ -4490,6 +4514,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_expand_mask_poly4_avx2 = sym super::mldsa::vg_mldsa_expand_mask_poly4_avx2, vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, @@ -4896,26 +4921,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 29b", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2592], al", "mov eax, 0", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2626], al", "mov eax, 0", "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", + "mov eax, 6", "mov BYTE PTR [rbx+2660], al", "mov eax, 0", "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", "mov eax, 0", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -4923,7 +4948,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", + "add rdi, 32768", "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", @@ -4932,19 +4957,19 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 210b", - "mov eax, 0", + "mov eax, 1", "mov BYTE PTR [rbx+2592], al", "mov eax, 1", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+2626], al", "mov eax, 1", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2660], al", "mov eax, 1", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2694], al", "mov eax, 1", "mov BYTE PTR [rbx+2695], al", @@ -4968,26 +4993,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 211b", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+2592], al", "mov eax, 1", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", + "mov eax, 6", "mov BYTE PTR [rbx+2626], al", "mov eax, 1", "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", + "mov eax, 0", "mov BYTE PTR [rbx+2660], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", "mov eax, 1", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -4995,7 +5020,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", + "add rdi, 40960", "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", @@ -5004,19 +5029,19 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 212b", - "mov eax, 0", + "mov eax, 2", "mov BYTE PTR [rbx+2592], al", "mov eax, 2", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 3", "mov BYTE PTR [rbx+2626], al", "mov eax, 2", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+2660], al", "mov eax, 2", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+2694], al", "mov eax, 2", "mov BYTE PTR [rbx+2695], al", @@ -5040,26 +5065,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 213b", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+2592], al", "mov eax, 2", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", + "mov eax, 0", "mov BYTE PTR [rbx+2626], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", + "mov eax, 1", "mov BYTE PTR [rbx+2660], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", "mov eax, 2", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -5067,7 +5092,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", + "add rdi, 49152", "mov ecx, 1024", "214:", "mov eax, DWORD PTR [rdi]", @@ -5076,19 +5101,19 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 214b", - "mov eax, 0", + "mov eax, 3", "mov BYTE PTR [rbx+2592], al", "mov eax, 3", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+2626], al", "mov eax, 3", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+2660], al", "mov eax, 3", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+2694], al", "mov eax, 3", "mov BYTE PTR [rbx+2695], al", @@ -5112,26 +5137,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 215b", - "mov eax, 3", + "mov eax, 0", "mov BYTE PTR [rbx+2592], al", - "mov eax, 3", - "mov BYTE PTR [rbx+2593], al", "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", "mov BYTE PTR [rbx+2626], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", + "mov eax, 2", "mov BYTE PTR [rbx+2660], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", "mov eax, 3", - "mov BYTE PTR [rbx+2695], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -5139,7 +5164,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", + "add rdi, 57344", "mov ecx, 1024", "216:", "mov eax, DWORD PTR [rdi]", @@ -5148,21 +5173,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 216b", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+2592], al", "mov eax, 4", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+2626], al", "mov eax, 4", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+2660], al", "mov eax, 4", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 0", "mov BYTE PTR [rbx+2694], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", @@ -5184,26 +5209,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 217b", - "mov eax, 3", + "mov eax, 1", "mov BYTE PTR [rbx+2592], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", + "mov eax, 2", "mov BYTE PTR [rbx+2626], al", - "mov eax, 4", - "mov BYTE PTR [rbx+2627], al", "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 3", "mov BYTE PTR [rbx+2660], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", "mov eax, 4", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 65536", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -5211,7 +5236,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 64512", + "add rdi, 65536", "mov ecx, 1024", "218:", "mov eax, DWORD PTR [rdi]", @@ -5220,21 +5245,21 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 218b", - "mov eax, 0", + "mov eax, 5", "mov BYTE PTR [rbx+2592], al", "mov eax, 5", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 6", "mov BYTE PTR [rbx+2626], al", "mov eax, 5", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 0", "mov BYTE PTR [rbx+2660], al", - "mov eax, 5", + "mov eax, 6", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 1", "mov BYTE PTR [rbx+2694], al", - "mov eax, 5", + "mov eax, 6", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", @@ -5256,62 +5281,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 219b", - "mov eax, 3", - "mov BYTE PTR [rbx+2592], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", - "mov BYTE PTR [rbx+2626], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2660], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2695], al", - "mov rdi, rbx", - "add rdi, 2560", - "mov rsi, rbx", - "add rsi, 72704", - "mov rdx, rbx", - "add rdx, 94208", - "call {vg_mldsa_rej_ntt_poly4_avx2}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 1024", - "220:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 220b", - "mov eax, 0", + "mov eax, 2", "mov BYTE PTR [rbx+2592], al", "mov eax, 6", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 3", "mov BYTE PTR [rbx+2626], al", "mov eax, 6", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+2660], al", "mov eax, 6", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+2694], al", "mov eax, 6", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -5319,71 +5308,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 77824", + "add rdi, 73728", "mov ecx, 1024", - "221:", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", - "mov eax, 3", + "jne 220b", + "mov eax, 6", "mov BYTE PTR [rbx+2592], al", "mov eax, 6", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", - "mov BYTE PTR [rbx+2626], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2660], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2695], al", - "mov rdi, rbx", - "add rdi, 2560", - "mov rsi, rbx", - "add rsi, 80896", - "mov rdx, rbx", - "add rdx, 94208", - "call {vg_mldsa_rej_ntt_poly4_avx2}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 80896", - "mov ecx, 1024", - "222:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 222b", "mov eax, 0", - "mov BYTE PTR [rbx+2592], al", - "mov eax, 7", - "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", "mov BYTE PTR [rbx+2626], al", "mov eax, 7", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 1", "mov BYTE PTR [rbx+2660], al", "mov eax, 7", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 2", "mov BYTE PTR [rbx+2694], al", "mov eax, 7", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 86016", + "add rsi, 77824", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -5391,15 +5344,15 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 86016", + "add rdi, 77824", "mov ecx, 1024", - "223:", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", + "jne 221b", "mov eax, 3", "mov BYTE PTR [rbx+2592], al", "mov eax, 7", @@ -5419,7 +5372,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 81920", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4_avx2}", @@ -5427,15 +5380,15 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 89088", + "add rdi, 81920", "mov ecx, 1024", - "224:", + "222:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", + "jne 222b", "mov rdi, r13", "add rdi, 0", "mov esi, 64", @@ -5451,13 +5404,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "225:", + "223:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", + "jne 223b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -5592,49 +5545,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -5683,49 +5636,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 51200", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -5774,49 +5727,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -5865,49 +5818,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 59392", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 60416", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 61440", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 66560", + "add rsi, 62464", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 67584", + "add rsi, 63488", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -5956,49 +5909,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 64512", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 65536", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 66560", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 67584", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 68608", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 69632", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 70656", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -6047,49 +6000,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 71680", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 78848", + "add rsi, 72704", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 74752", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 75776", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 82944", + "add rsi, 76800", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 77824", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -6138,49 +6091,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 86016", + "add rsi, 78848", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 79872", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 88064", + "add rsi, 80896", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 81920", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 90112", + "add rsi, 82944", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 91136", + "add rsi, 83968", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 92160", + "add rsi, 84992", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", @@ -6296,7 +6249,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 0", "mov ecx, 64", "mov edx, 0", - "226:", + "224:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -6304,7 +6257,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 226b", + "jne 224b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -9292,121 +9245,145 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov eax, 814", "mov QWORD PTR [rbx+888], rax", "27:", + "mov rdi, rbx", + "add rdi, 1296", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "28:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 28b", "mov rax, QWORD PTR [rbx+896]", "add rax, 0", - "mov BYTE PTR [rbx+1024], al", + "mov BYTE PTR [rbx+1360], al", "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", + "mov BYTE PTR [rbx+1361], al", "mov rdi, rbx", - "add rdi, 960", + "add rdi, 1362", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "29:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 29b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 1", + "mov BYTE PTR [rbx+1426], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1427], al", + "mov rdi, rbx", + "add rdi, 1428", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "210:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 210b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 2", + "mov BYTE PTR [rbx+1492], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1493], al", + "mov rdi, rbx", + "add rdi, 1494", + "mov rsi, rbx", + "add rsi, 960", + "mov ecx, 8", + "211:", + "mov rax, QWORD PTR [rsi]", + "mov QWORD PTR [rdi], rax", + "add rdi, 8", + "add rsi, 8", + "sub rcx, 1", + "jne 211b", + "mov rax, QWORD PTR [rbx+896]", + "add rax, 3", + "mov BYTE PTR [rbx+1558], al", + "shr rax, 8", + "mov BYTE PTR [rbx+1559], al", + "mov rdi, rbx", + "add rdi, 1296", "mov esi, 524288", "mov rdx, rbx", "add rdx, 18432", "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", + "add rcx, 121856", + "call {vg_mldsa_expand_mask_poly4}", "mov rdi, rbx", "add rdi, 25600", "mov rsi, rbx", "add rsi, 18432", "mov ecx, 128", - "28:", + "212:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 28b", + "jne 212b", "mov rdi, rbx", "add rdi, 25600", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 1", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 19456", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", "add rsi, 19456", "mov ecx, 128", - "29:", + "213:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 29b", + "jne 213b", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 2", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 20480", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 27648", "mov rsi, rbx", "add rsi, 20480", "mov ecx, 128", - "210:", + "214:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 210b", + "jne 214b", "mov rdi, rbx", "add rdi, 27648", "mov rsi, rbx", "add rsi, 3072", "call {vg_mldsa_ntt}", - "mov rax, QWORD PTR [rbx+896]", - "add rax, 3", - "mov BYTE PTR [rbx+1024], al", - "shr rax, 8", - "mov BYTE PTR [rbx+1025], al", - "mov rdi, rbx", - "add rdi, 960", - "mov esi, 524288", - "mov rdx, rbx", - "add rdx, 21504", - "mov rcx, rbx", - "add rcx, 3072", - "call {vg_mldsa_expand_mask_poly}", "mov rdi, rbx", "add rdi, 28672", "mov rsi, rbx", "add rsi, 21504", "mov ecx, 128", - "211:", + "215:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 211b", + "jne 215b", "mov rdi, rbx", "add rdi, 28672", "mov rsi, rbx", @@ -9430,13 +9407,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 22528", "mov ecx, 128", - "212:", + "216:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 212b", + "jne 216b", "mov rdi, rbx", "add rdi, 29696", "mov rsi, rbx", @@ -9460,13 +9437,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 23552", "mov ecx, 128", - "213:", + "217:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 213b", + "jne 217b", "mov rdi, rbx", "add rdi, 30720", "mov rsi, rbx", @@ -9490,13 +9467,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 24576", "mov ecx, 128", - "214:", + "218:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 214b", + "jne 218b", "mov rdi, rbx", "add rdi, 31744", "mov rsi, rbx", @@ -10112,12 +10089,12 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "add r8, 3072", "call {vg_mldsa_sample_in_ball}", "test eax, eax", - "jne 215f", + "jne 219f", "mov r15d, 0", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "jmp 216f", - "215:", + "jmp 220f", + "219:", "mov rdi, rbx", "add rdi, 5120", "mov rsi, rbx", @@ -10526,13 +10503,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 32768", "mov ecx, 128", - "217:", + "221:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 217b", + "jne 221b", "mov rdi, rbx", "add rdi, 32768", "mov rsi, rbx", @@ -10576,13 +10553,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 33792", "mov ecx, 128", - "218:", + "222:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 218b", + "jne 222b", "mov rdi, rbx", "add rdi, 33792", "mov rsi, rbx", @@ -10626,13 +10603,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 34816", "mov ecx, 128", - "219:", + "223:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 219b", + "jne 223b", "mov rdi, rbx", "add rdi, 34816", "mov rsi, rbx", @@ -10676,13 +10653,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 35840", "mov ecx, 128", - "220:", + "224:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 220b", + "jne 224b", "mov rdi, rbx", "add rdi, 35840", "mov rsi, rbx", @@ -10726,13 +10703,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 36864", "mov ecx, 128", - "221:", + "225:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 221b", + "jne 225b", "mov rdi, rbx", "add rdi, 36864", "mov rsi, rbx", @@ -10776,13 +10753,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 37888", "mov ecx, 128", - "222:", + "226:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 222b", + "jne 226b", "mov rdi, rbx", "add rdi, 37888", "mov rsi, rbx", @@ -10826,13 +10803,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 38912", "mov ecx, 128", - "223:", + "227:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 223b", + "jne 227b", "mov rdi, rbx", "add rdi, 38912", "mov rsi, rbx", @@ -10876,13 +10853,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "mov rsi, rbx", "add rsi, 39936", "mov ecx, 128", - "224:", + "228:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 224b", + "jne 228b", "mov rdi, rbx", "add rdi, 39936", "mov rsi, rbx", @@ -10909,36 +10886,36 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "shr rax, 63", "and r15d, eax", "test r15d, r15d", - "jne 225f", + "jne 229f", "mov rax, QWORD PTR [rbx+896]", "add rax, 7", "mov QWORD PTR [rbx+896], rax", - "jmp 226f", - "225:", + "jmp 230f", + "229:", "mov eax, 1", "mov QWORD PTR [rbx+888], rax", - "226:", - "216:", + "230:", + "220:", "mov rax, QWORD PTR [rbx+888]", "sub rax, 1", "mov QWORD PTR [rbx+888], rax", "jne 27b", "test r15d, r15d", - "jne 227f", - "jmp 228f", - "227:", + "jne 231f", + "jmp 232f", + "231:", "mov rdi, r14", "add rdi, 0", "mov rsi, rbx", "add rsi, 1040", "mov ecx, 8", - "229:", + "233:", "mov rax, QWORD PTR [rsi]", "mov QWORD PTR [rdi], rax", "add rdi, 8", "add rsi, 8", "sub rcx, 1", - "jne 229b", + "jne 233b", "mov rdi, rbx", "add rdi, 18432", "mov esi, 524287", @@ -11003,7 +10980,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: "add rcx, 4544", "mov r8d, 83", "call {vg_mldsa_hint_bit_pack}", - "228:", + "232:", "26:", "mov eax, r15d", "mov r15, QWORD PTR [rbx+880]", @@ -11019,6 +10996,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, vg_keccak_pad = sym super::sha3::vg_keccak_pad, vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + vg_mldsa_expand_mask_poly4 = sym super::mldsa::vg_mldsa_expand_mask_poly4, vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, vg_mldsa_multiply_add_ntt = sym super::mldsa::vg_mldsa_multiply_add_ntt, @@ -11417,26 +11395,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rdi, 4", "sub rcx, 1", "jne 29b", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2592], al", "mov eax, 0", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2626], al", "mov eax, 0", "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", + "mov eax, 6", "mov BYTE PTR [rbx+2660], al", "mov eax, 0", "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", "mov eax, 0", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11444,7 +11422,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", + "add rdi, 32768", "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", @@ -11453,19 +11431,19 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rdi, 4", "sub rcx, 1", "jne 210b", - "mov eax, 0", + "mov eax, 1", "mov BYTE PTR [rbx+2592], al", "mov eax, 1", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+2626], al", "mov eax, 1", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2660], al", "mov eax, 1", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2694], al", "mov eax, 1", "mov BYTE PTR [rbx+2695], al", @@ -11489,26 +11467,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rdi, 4", "sub rcx, 1", "jne 211b", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+2592], al", "mov eax, 1", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", + "mov eax, 6", "mov BYTE PTR [rbx+2626], al", "mov eax, 1", "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", + "mov eax, 0", "mov BYTE PTR [rbx+2660], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", "mov eax, 1", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11516,7 +11494,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", + "add rdi, 40960", "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", @@ -11525,19 +11503,19 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rdi, 4", "sub rcx, 1", "jne 212b", - "mov eax, 0", + "mov eax, 2", "mov BYTE PTR [rbx+2592], al", "mov eax, 2", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 3", "mov BYTE PTR [rbx+2626], al", "mov eax, 2", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+2660], al", "mov eax, 2", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+2694], al", "mov eax, 2", "mov BYTE PTR [rbx+2695], al", @@ -11561,62 +11539,26 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rdi, 4", "sub rcx, 1", "jne 213b", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+2592], al", "mov eax, 2", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", - "mov BYTE PTR [rbx+2626], al", - "mov eax, 2", - "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2660], al", - "mov eax, 2", - "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", - "mov eax, 2", - "mov BYTE PTR [rbx+2695], al", - "mov rdi, rbx", - "add rdi, 2560", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 94208", - "call {vg_mldsa_rej_ntt_poly4}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 1024", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", "mov eax, 0", - "mov BYTE PTR [rbx+2592], al", - "mov eax, 3", - "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", "mov BYTE PTR [rbx+2626], al", "mov eax, 3", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 1", "mov BYTE PTR [rbx+2660], al", "mov eax, 3", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 2", "mov BYTE PTR [rbx+2694], al", "mov eax, 3", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11624,15 +11566,15 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", + "add rdi, 49152", "mov ecx, 1024", - "215:", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 215b", + "jne 214b", "mov eax, 3", "mov BYTE PTR [rbx+2592], al", "mov eax, 3", @@ -11652,7 +11594,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11660,15 +11602,15 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", + "add rdi, 53248", "mov ecx, 1024", - "216:", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", + "jne 215b", "mov eax, 0", "mov BYTE PTR [rbx+2592], al", "mov eax, 4", @@ -11688,7 +11630,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11696,35 +11638,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 61440", + "add rdi, 57344", "mov ecx, 1024", - "217:", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 217b", - "mov eax, 3", + "jne 216b", + "mov eax, 4", "mov BYTE PTR [rbx+2592], al", "mov eax, 4", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2626], al", "mov eax, 4", "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", + "mov eax, 6", "mov BYTE PTR [rbx+2660], al", "mov eax, 4", "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", + "mov eax, 0", "mov BYTE PTR [rbx+2694], al", - "mov eax, 4", + "mov eax, 5", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 61440", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11732,35 +11674,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 64512", + "add rdi, 61440", "mov ecx, 1024", - "218:", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 218b", - "mov eax, 0", + "jne 217b", + "mov eax, 1", "mov BYTE PTR [rbx+2592], al", "mov eax, 5", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+2626], al", "mov eax, 5", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+2660], al", "mov eax, 5", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+2694], al", "mov eax, 5", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 65536", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11768,71 +11710,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 69632", + "add rdi, 65536", "mov ecx, 1024", - "219:", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 219b", - "mov eax, 3", + "jne 218b", + "mov eax, 5", "mov BYTE PTR [rbx+2592], al", "mov eax, 5", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", + "mov eax, 6", "mov BYTE PTR [rbx+2626], al", "mov eax, 5", "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2660], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2694], al", - "mov eax, 5", - "mov BYTE PTR [rbx+2695], al", - "mov rdi, rbx", - "add rdi, 2560", - "mov rsi, rbx", - "add rsi, 72704", - "mov rdx, rbx", - "add rdx, 94208", - "call {vg_mldsa_rej_ntt_poly4}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 1024", - "220:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 220b", "mov eax, 0", - "mov BYTE PTR [rbx+2592], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", - "mov BYTE PTR [rbx+2626], al", - "mov eax, 6", - "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", "mov BYTE PTR [rbx+2660], al", "mov eax, 6", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 1", "mov BYTE PTR [rbx+2694], al", "mov eax, 6", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 69632", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11840,35 +11746,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 77824", + "add rdi, 69632", "mov ecx, 1024", - "221:", + "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", - "mov eax, 3", + "jne 219b", + "mov eax, 2", "mov BYTE PTR [rbx+2592], al", "mov eax, 6", "mov BYTE PTR [rbx+2593], al", - "mov eax, 4", + "mov eax, 3", "mov BYTE PTR [rbx+2626], al", "mov eax, 6", "mov BYTE PTR [rbx+2627], al", - "mov eax, 5", + "mov eax, 4", "mov BYTE PTR [rbx+2660], al", "mov eax, 6", "mov BYTE PTR [rbx+2661], al", - "mov eax, 6", + "mov eax, 5", "mov BYTE PTR [rbx+2694], al", "mov eax, 6", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11876,35 +11782,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 80896", + "add rdi, 73728", "mov ecx, 1024", - "222:", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", - "mov eax, 0", + "jne 220b", + "mov eax, 6", "mov BYTE PTR [rbx+2592], al", - "mov eax, 7", + "mov eax, 6", "mov BYTE PTR [rbx+2593], al", - "mov eax, 1", + "mov eax, 0", "mov BYTE PTR [rbx+2626], al", "mov eax, 7", "mov BYTE PTR [rbx+2627], al", - "mov eax, 2", + "mov eax, 1", "mov BYTE PTR [rbx+2660], al", "mov eax, 7", "mov BYTE PTR [rbx+2661], al", - "mov eax, 3", + "mov eax, 2", "mov BYTE PTR [rbx+2694], al", "mov eax, 7", "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 86016", + "add rsi, 77824", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11912,15 +11818,15 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 86016", + "add rdi, 77824", "mov ecx, 1024", - "223:", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", + "jne 221b", "mov eax, 3", "mov BYTE PTR [rbx+2592], al", "mov eax, 7", @@ -11940,7 +11846,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 2560", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 81920", "mov rdx, rbx", "add rdx, 94208", "call {vg_mldsa_rej_ntt_poly4}", @@ -11948,15 +11854,15 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 89088", + "add rdi, 81920", "mov ecx, 1024", - "224:", + "222:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", + "jne 222b", "mov rdi, r13", "add rdi, 0", "mov esi, 64", @@ -11972,13 +11878,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "225:", + "223:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", + "jne 223b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -12113,49 +12019,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt}", @@ -12204,49 +12110,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 51200", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt}", @@ -12295,49 +12201,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt}", @@ -12386,49 +12292,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 59392", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 60416", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 61440", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 66560", + "add rsi, 62464", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 67584", + "add rsi, 63488", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt}", @@ -12477,49 +12383,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 64512", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 65536", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 66560", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 67584", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 68608", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 69632", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 70656", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt}", @@ -12568,49 +12474,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 71680", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 78848", + "add rsi, 72704", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 74752", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 75776", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 82944", + "add rsi, 76800", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 77824", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt}", @@ -12659,49 +12565,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 86016", + "add rsi, 78848", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 79872", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 88064", + "add rsi, 80896", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 81920", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 90112", + "add rsi, 82944", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 91136", + "add rsi, 83968", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 92160", + "add rsi, 84992", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt}", @@ -12817,7 +12723,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rdi, 0", "mov ecx, 64", "mov edx, 0", - "226:", + "224:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -12825,7 +12731,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 226b", + "jne 224b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax",