diff --git a/Cargo.lock b/Cargo.lock index 431d53f25..091756787 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2013,6 +2013,7 @@ dependencies = [ "postcard", "rand 0.10.1", "rayon", + "silver_crypto_simd", "thiserror 2.0.18", "tracing", "zk_alloc", @@ -2159,6 +2160,7 @@ dependencies = [ "serde_json", "serde_yaml_ng", "sha2", + "silver_crypto_simd", "snap", "thiserror 2.0.18", "tracing", @@ -6877,6 +6879,15 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "silver_crypto_simd" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df4b23a7896834e59c57593b90d65ffc3d5bc3236da7a74d259dd7ec45108f2" +dependencies = [ + "blst", +] + [[package]] name = "simd-adler32" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index e13475448..f0539c3b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,6 +171,10 @@ vergen-git2 = { version = "9", features = ["rustc"] } git2 = { version = "0.20", default-features = false } rayon = "1.11" +# BLS12-381 decoding and pairings plus PeerDAS cell checks and extension, on +# AVX-512 IFMA where the CPU has it (runtime detection, no build flags). Used by +# `ethlambda-crypto`'s `bls` and `ethlambda-state-transition`'s `beacon::kzg`. +silver_crypto_simd = "0.1.0" rand = "0.10" lru = "0.16" rocksdb = "0.24" diff --git a/crates/blockchain/state_transition/Cargo.toml b/crates/blockchain/state_transition/Cargo.toml index 7b819c538..05f6e86b4 100644 --- a/crates/blockchain/state_transition/Cargo.toml +++ b/crates/blockchain/state_transition/Cargo.toml @@ -83,6 +83,10 @@ rayon.workspace = true # version and feature set ethrex uses, plus the embedded mainnet trusted setup. c-kzg = { version = "2.1.8", features = ["ethereum_kzg_settings"] } +# `beacon::kzg`'s cell extension and cell-proof batch check, on AVX-512 IFMA +# where the CPU has it; everything else there stays on c-kzg. +silver_crypto_simd.workspace = true + # Reduction modulo the BLS field order, for the two Fiat-Shamir challenge # functions that c-kzg does not export. num-bigint = "0.4.6" diff --git a/crates/blockchain/state_transition/src/beacon/data_columns.rs b/crates/blockchain/state_transition/src/beacon/data_columns.rs index 26cc347e8..571f313ae 100644 --- a/crates/blockchain/state_transition/src/beacon/data_columns.rs +++ b/crates/blockchain/state_transition/src/beacon/data_columns.rs @@ -151,8 +151,8 @@ pub fn get_data_column_sidecars( let column: Vec = cells_and_proofs .iter() .map(|(cells, _)| { - SszVector::try_from(cells[column_index].to_bytes().to_vec()) - .expect("a c-kzg cell is BYTES_PER_CELL bytes") + SszVector::try_from(cells[column_index].to_vec()) + .expect("a cell is BYTES_PER_CELL bytes") }) .collect(); let kzg_proofs: Vec = cells_and_proofs diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 2af9bf4f5..243b36b23 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -4174,14 +4174,18 @@ fn verify_column_cells( let cell_indices = vec![index; column.len()]; let mut cells = Vec::with_capacity(column.len()); for cell in column { - cells.push( - c_kzg::Cell::from_bytes(&cell[..]) - .map_err(|_| Error::SpecAssert("len(cell) == BYTES_PER_CELL"))?, - ); + cells.push(kzg_cell(cell)?); } kzg::verify_cell_kzg_proof_batch(commitments, &cell_indices, &cells, proofs) } +/// A column's cell as the KZG module's [`kzg::Cell`]. +fn kzg_cell(cell: &fulu::Cell) -> Result { + cell[..] + .try_into() + .map_err(|_| Error::SpecAssert("len(cell) == BYTES_PER_CELL")) +} + /// One column's worth of input to [`verify_column_cells_batch`]: the cell /// position every cell of the column sits at, the cells, and the commitments /// and proofs they are checked against. @@ -4210,10 +4214,7 @@ pub fn verify_column_cells_batch(columns: &[ColumnCells<'_>]) -> Result { for column in columns { cell_indices.extend(std::iter::repeat_n(column.index, column.column.len())); for cell in column.column { - cells.push( - c_kzg::Cell::from_bytes(&cell[..]) - .map_err(|_| Error::SpecAssert("len(cell) == BYTES_PER_CELL"))?, - ); + cells.push(kzg_cell(cell)?); } commitments.extend_from_slice(column.commitments); proofs.extend_from_slice(column.proofs); diff --git a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs index b88f07174..24277f595 100644 --- a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs +++ b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs @@ -320,7 +320,7 @@ pub fn gloas_data_column_sidecars( let mut kzg_proofs = Vec::with_capacity(blobs.len()); for (blob, cells) in per_blob_cells.iter().enumerate() { let cell = ethlambda_types::beacon::containers::fulu::Cell::try_from( - cells[column_index].to_bytes().to_vec(), + cells[column_index].to_vec(), ) .map_err(|_| Error::SpecAssert("len(cell) == BYTES_PER_CELL"))?; column.push(cell); diff --git a/crates/blockchain/state_transition/src/beacon/gossip/column.rs b/crates/blockchain/state_transition/src/beacon/gossip/column.rs index e45808eae..f42065011 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/column.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/column.rs @@ -1766,10 +1766,8 @@ mod tests { let column: Vec = cells .iter() .map(|blob_cells| { - libssz_types::SszVector::try_from( - blob_cells[index as usize].to_bytes().to_vec(), - ) - .expect("exact cell size") + libssz_types::SszVector::try_from(blob_cells[index as usize].to_vec()) + .expect("exact cell size") }) .collect(); let column_proofs: Vec = proofs diff --git a/crates/blockchain/state_transition/src/beacon/kzg.rs b/crates/blockchain/state_transition/src/beacon/kzg.rs index 3b20ccd02..f9d8ad71b 100644 --- a/crates/blockchain/state_transition/src/beacon/kzg.rs +++ b/crates/blockchain/state_transition/src/beacon/kzg.rs @@ -18,6 +18,17 @@ //! `verify_cell_kzg_proof_batch` and does not export them, so they are //! reimplemented here directly from the spec text. //! +//! Two functions run on [`silver_crypto_simd`] instead of c-kzg: +//! [`compute_cells`] and [`verify_cell_kzg_proof_batch`], the cell extension +//! and the cell-proof batch check every data column goes through. On an x86-64 +//! CPU with AVX-512 IFMA they work on eight field elements, curve points or +//! Miller loops at a time; elsewhere the same API runs portable Rust and blst. +//! Both carry Ethereum's trusted setup themselves and return the +//! specification's verdict. Everything else here (commitments, blob proofs, +//! cell proofs and recovery) stays on c-kzg, which `silver_crypto_simd` does +//! not replace. A cell is therefore this module's own [`Cell`], a byte array, +//! converted to c-kzg's type only at the calls into c-kzg. +//! //! Every function accepts untrusted bytes and returns [`Result`] rather than //! panicking. The deneb spec says as much of its own public functions: they //! "MUST accept raw bytes as input and perform the required cryptographic @@ -29,19 +40,91 @@ //! rejection directly: many cases have `output: null`, meaning the operation //! must fail rather than panic. -use std::sync::LazyLock; +use std::sync::{LazyLock, Mutex, PoisonError}; use num_bigint::BigUint; +use silver_crypto_simd::bls::{COMPRESSED_LEN, uncompress_in_g1}; +use silver_crypto_simd::kzg::{ + BYTES_PER_BLOB, BYTES_PER_CELL, BYTES_PER_FIELD_ELEMENT, CELLS_PER_EXT_BLOB, CellBatchVerifier, + CellExtender, +}; use crate::beacon::error::{Error, Result, verify}; use crate::beacon::hash::hash; use crate::beacon::primitives::{Bytes32, H256, KzgCommitment, KzgProof}; +// The two libraries must agree on the shapes this module converts between. +const _: () = assert!(BYTES_PER_CELL == c_kzg::BYTES_PER_CELL); +const _: () = assert!(CELLS_PER_EXT_BLOB == c_kzg::CELLS_PER_EXT_BLOB); +const _: () = assert!(BYTES_PER_BLOB == c_kzg::BYTES_PER_BLOB); + +/// One cell of an extended blob: the fulu spec's `Cell`, `BYTES_PER_CELL` +/// bytes holding `FIELD_ELEMENTS_PER_CELL` big-endian field elements. +pub type Cell = [u8; BYTES_PER_CELL]; /// The cells (equivalently, proofs) produced from one extended blob: the /// fulu spec's `CELLS_PER_EXT_BLOB`-length vectors. -pub type CellsPerExtBlob = [c_kzg::Cell; c_kzg::CELLS_PER_EXT_BLOB]; +pub type CellsPerExtBlob = [Cell; CELLS_PER_EXT_BLOB]; /// A proof for each cell of an extended blob. -type ProofsPerExtBlob = [KzgProof; c_kzg::CELLS_PER_EXT_BLOB]; +type ProofsPerExtBlob = [KzgProof; CELLS_PER_EXT_BLOB]; + +/// The most idle instances a [`Pool`] keeps. More than this many callers at +/// once still each get an instance; the extras are dropped on return. +const POOL_MAX_IDLE: usize = 16; + +/// The largest batch, in cells, whose [`CellBatchVerifier`] goes back to its +/// pool. A verifier keeps its buffers at the size of the largest batch it has +/// checked, so one that checked a whole block's cells is dropped rather than +/// kept: gossip checks one column at a time, which is far below this. +const MAX_RETAINED_BATCH_CELLS: usize = 256; + +/// Reusable instances of a type that is costly to build and keeps buffers +/// between uses: `silver_crypto_simd`'s cell verifier and cell extender, which +/// precompute their domains and setup on construction and take `&mut self`. +/// +/// A caller takes an idle instance or builds a new one, and gives it back when +/// done, so concurrent callers (gossip's blocking threads, rayon's workers) +/// wait on one another only for a `Vec` push or pop, never for a computation. +struct Pool { + idle: Mutex>, + build: fn() -> T, +} + +impl Pool { + const fn new(build: fn() -> T) -> Self { + Self { + idle: Mutex::new(Vec::new()), + build, + } + } + + /// Runs `f` on an idle instance, or a new one, and returns the instance to + /// the pool afterwards if `retain` is set and the pool has room. + /// + /// A panic in `f` drops the instance instead of returning it, and a + /// poisoned lock is still usable: the `Vec` is never left half-updated. + fn with(&self, retain: bool, f: impl FnOnce(&mut T) -> R) -> R { + let idle = self + .idle + .lock() + .unwrap_or_else(PoisonError::into_inner) + .pop(); + let mut instance = idle.unwrap_or_else(self.build); + let result = f(&mut instance); + if retain { + let mut idle = self.idle.lock().unwrap_or_else(PoisonError::into_inner); + if idle.len() < POOL_MAX_IDLE { + idle.push(instance); + } + } + result + } +} + +/// Cell-proof batch verifiers for [`verify_cell_kzg_proof_batch`]. +static CELL_VERIFIERS: Pool = Pool::new(CellBatchVerifier::new); + +/// Blob extenders for [`compute_cells`]. +static CELL_EXTENDERS: Pool = Pool::new(CellExtender::new); /// The Ethereum mainnet trusted setup, parsed once. /// @@ -74,6 +157,45 @@ static BLS_MODULUS: LazyLock = LazyLock::new(|| { .expect("BLS_MODULUS_DECIMAL is a valid base-10 literal") }); +/// `BLS_MODULUS` as a big-endian field element's bytes, for checking that an +/// element is canonical by comparing bytes. +static BLS_MODULUS_BYTES: LazyLock<[u8; BYTES_PER_FIELD_ELEMENT]> = LazyLock::new(|| { + BLS_MODULUS + .to_bytes_be() + .try_into() + .expect("BLS_MODULUS is exactly BYTES_PER_FIELD_ELEMENT bytes wide") +}); + +/// Whether every field element of `cell` is below `BLS_MODULUS`: the check +/// `cell_to_coset_evals` makes of each through `bytes_to_bls_field`. +fn cell_is_canonical(cell: &Cell) -> bool { + let (elements, _) = cell.as_chunks::(); + elements.iter().all(|element| element < &*BLS_MODULUS_BYTES) +} + +/// Whether every encoding is a G1 point the spec's `validate_kzg_g1` accepts: +/// the point at infinity, or a point of the prime-order subgroup. That is +/// exactly what `uncompress_in_g1` admits. +fn all_valid_kzg_g1(encodings: &[[u8; COMPRESSED_LEN]]) -> bool { + let mut decoded = vec![None; encodings.len()]; + uncompress_in_g1(encodings, &mut decoded); + decoded.iter().all(Option::is_some) +} + +/// This module's cells as c-kzg's, for the calls into c-kzg. +fn cells_to_c_kzg(cells: &[Cell]) -> Vec { + cells.iter().map(|cell| c_kzg::Cell::new(*cell)).collect() +} + +/// c-kzg's cells as this module's, kept on the heap throughout. +fn cells_from_c_kzg(raw: Box<[c_kzg::Cell; CELLS_PER_EXT_BLOB]>) -> Box { + let converted: Vec = raw.iter().map(c_kzg::Cell::to_bytes).collect(); + converted + .into_boxed_slice() + .try_into() + .expect("converted has exactly CELLS_PER_EXT_BLOB elements, one per input cell") +} + /// Parses raw bytes into a c-kzg blob, rejecting anything but exactly /// `BYTES_PER_BLOB` bytes. fn to_blob(bytes: &[u8]) -> Result { @@ -232,11 +354,20 @@ pub fn verify_blob_kzg_proof_batch( /// into `CELLS_PER_EXT_BLOB` cells, without the accompanying proofs; see /// [`compute_cells_and_kzg_proofs`] when the proofs are needed too. Fails if /// `blob` is invalid. +/// +/// Runs on `silver_crypto_simd`'s extender, whose only refusal is a field +/// element at or past `BLS_MODULUS`, exactly the blob the spec rejects. pub fn compute_cells(blob: &[u8]) -> Result> { - let blob = to_blob(blob)?; - KZG_SETTINGS - .compute_cells(&blob) - .map_err(|_| Error::SpecAssert("blob is the evaluation of a valid polynomial")) + let blob: &[u8; BYTES_PER_BLOB] = blob + .try_into() + .map_err(|_| Error::SpecAssert("len(blob) == BYTES_PER_BLOB"))?; + let mut cells: Box = vec![[0; BYTES_PER_CELL]; CELLS_PER_EXT_BLOB] + .into_boxed_slice() + .try_into() + .expect("the vector has exactly CELLS_PER_EXT_BLOB cells"); + let extended = CELL_EXTENDERS.with(true, |extender| extender.compute(blob, &mut cells)); + verify(extended, "blob is the evaluation of a valid polynomial")?; + Ok(cells) } /// The specification's `compute_cells_and_kzg_proofs` @@ -251,7 +382,7 @@ pub fn compute_cells_and_kzg_proofs( let (cells, proofs) = KZG_SETTINGS .compute_cells_and_kzg_proofs(&blob) .map_err(|_| Error::SpecAssert("blob is the evaluation of a valid polynomial"))?; - Ok((cells, proofs_from_c_kzg(proofs))) + Ok((cells_from_c_kzg(cells), proofs_from_c_kzg(proofs))) } /// The specification's `recover_cells_and_kzg_proofs` @@ -263,12 +394,12 @@ pub fn compute_cells_and_kzg_proofs( /// cells are given to recover the rest. pub fn recover_cells_and_kzg_proofs( cell_indices: &[u64], - cells: &[c_kzg::Cell], + cells: &[Cell], ) -> Result<(Box, Box)> { let (cells, proofs) = KZG_SETTINGS - .recover_cells_and_kzg_proofs(cell_indices, cells) + .recover_cells_and_kzg_proofs(cell_indices, &cells_to_c_kzg(cells)) .map_err(|_| Error::SpecAssert("cell_indices and cells recover a valid extended blob"))?; - Ok((cells, proofs_from_c_kzg(proofs))) + Ok((cells_from_c_kzg(cells), proofs_from_c_kzg(proofs))) } /// The specification's `verify_cell_kzg_proof_batch` @@ -279,25 +410,56 @@ pub fn recover_cells_and_kzg_proofs( /// `commitments`, `cell_indices`, `cells`, and `proofs` must have equal /// length; fails, rather than returning `false`, if they do not, or if any /// entry is malformed. +/// +/// Runs on `silver_crypto_simd`'s batch verifier, which answers `true` +/// exactly when the spec does and `false` for everything else, malformed +/// input included. So a `false` is followed by the spec's own input +/// assertions, which tell a malformed batch (`Err`) from a well-formed one +/// that does not verify (`Ok(false)`). That second pass runs only on a +/// rejection, which a valid column never meets. pub fn verify_cell_kzg_proof_batch( commitments: &[KzgCommitment], cell_indices: &[u64], - cells: &[c_kzg::Cell], + cells: &[Cell], proofs: &[KzgProof], ) -> Result { - let commitment_bytes: Vec = commitments - .iter() - .map(|commitment| c_kzg::Bytes48::new(commitment.0)) - .collect(); - let proof_bytes: Vec = proofs - .iter() - .map(|proof| c_kzg::Bytes48::new(proof.0)) - .collect(); - KZG_SETTINGS - .verify_cell_kzg_proof_batch(&commitment_bytes, cell_indices, cells, &proof_bytes) - .map_err(|_| { - Error::SpecAssert("commitments, cell_indices, cells, and proofs are consistent") - }) + let commitment_bytes: Vec<[u8; COMPRESSED_LEN]> = + commitments.iter().map(|commitment| commitment.0).collect(); + let proof_bytes: Vec<[u8; COMPRESSED_LEN]> = proofs.iter().map(|proof| proof.0).collect(); + let retain = cells.len() <= MAX_RETAINED_BATCH_CELLS; + let verified = CELL_VERIFIERS.with(retain, |verifier| { + verifier.verify(&commitment_bytes, cell_indices, cells, &proof_bytes) + }); + if verified { + return Ok(true); + } + + // The assertions `verify_cell_kzg_proof_batch` makes of its inputs, in + // its own order; any that fails makes the call an error, not a `false`. + let n = cells.len(); + verify( + commitment_bytes.len() == n && proof_bytes.len() == n && cell_indices.len() == n, + "len(commitments_bytes) == len(cells) == len(proofs_bytes) == len(cell_indices)", + )?; + verify( + cell_indices + .iter() + .all(|&index| index < CELLS_PER_EXT_BLOB as u64), + "cell_index < CELLS_PER_EXT_BLOB", + )?; + verify( + all_valid_kzg_g1(&commitment_bytes), + "bytes_to_kzg_commitment(commitment_bytes)", + )?; + verify( + cells.iter().all(cell_is_canonical), + "cell_to_coset_evals(cell)", + )?; + verify( + all_valid_kzg_g1(&proof_bytes), + "bytes_to_kzg_proof(proof_bytes)", + )?; + Ok(false) } /// Domain separator for `compute_challenge`: the deneb spec's @@ -528,10 +690,10 @@ mod tests { Some(KzgProof(bytes)) } - /// Decodes a `BYTES_PER_CELL`-byte hex string into a [`c_kzg::Cell`], or - /// `None` if the length is wrong. - fn hex_cell(value: &Value) -> Option { - c_kzg::Cell::from_bytes(&hex_bytes(value)).ok() + /// Decodes a `BYTES_PER_CELL`-byte hex string into a [`Cell`], or `None` + /// if the length is wrong. + fn hex_cell(value: &Value) -> Option { + hex_bytes(value).try_into().ok() } #[test] @@ -790,7 +952,7 @@ mod tests { .iter() .map(|v| v.as_u64().expect("cell index is an integer")) .collect(); - let cells: Option> = input["cells"] + let cells: Option> = input["cells"] .as_sequence() .expect("cells is a list") .iter() @@ -846,7 +1008,7 @@ mod tests { .iter() .map(hex_commitment) .collect(); - let cells: Option> = input["cells"] + let cells: Option> = input["cells"] .as_sequence() .expect("cells is a list") .iter() @@ -970,4 +1132,151 @@ mod tests { cases.len() ); } + + /// A blob of canonical field elements, distinct per `seed`: every + /// element's top byte is cleared below `BLS_MODULUS`'s. + fn test_blob(seed: u64) -> Vec { + let mut state = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + let mut blob = vec![0u8; BYTES_PER_BLOB]; + for chunk in blob.chunks_mut(8) { + // xorshift64*: deterministic, and enough to make the cells distinct. + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + chunk.copy_from_slice(&state.wrapping_mul(0x2545_f491_4f6c_dd1d).to_le_bytes()); + } + for element in blob.chunks_mut(BYTES_PER_FIELD_ELEMENT) { + element[0] &= 0x3f; + } + blob + } + + /// c-kzg's answer on a batch, as `Ok(verdict)` or `Err(())`. + fn c_kzg_verdict( + commitments: &[KzgCommitment], + cell_indices: &[u64], + cells: &[Cell], + proofs: &[KzgProof], + ) -> std::result::Result { + let commitments: Vec = commitments + .iter() + .map(|commitment| c_kzg::Bytes48::new(commitment.0)) + .collect(); + let proofs: Vec = proofs + .iter() + .map(|proof| c_kzg::Bytes48::new(proof.0)) + .collect(); + KZG_SETTINGS + .verify_cell_kzg_proof_batch( + &commitments, + cell_indices, + &cells_to_c_kzg(cells), + &proofs, + ) + .map_err(|_| ()) + } + + /// One batch to judge: commitments, cell indices, cells and proofs. + type Batch = (Vec, Vec, Vec, Vec); + + /// [`compute_cells`] (on `silver_crypto_simd`) produces c-kzg's cells for + /// valid blobs, and both refuse a blob with a non-canonical element. + #[test] + fn compute_cells_agrees_with_c_kzg() { + for seed in 0..3 { + let blob = test_blob(seed); + let ours = compute_cells(&blob).unwrap(); + let theirs = KZG_SETTINGS + .compute_cells(&to_blob(&blob).unwrap()) + .unwrap(); + assert_eq!(*ours, *cells_from_c_kzg(theirs), "blob {seed}"); + } + let mut invalid = test_blob(9); + invalid[BYTES_PER_FIELD_ELEMENT * 17] = 0xff; + assert!(compute_cells(&invalid).is_err()); + assert!( + KZG_SETTINGS + .compute_cells(&to_blob(&invalid).unwrap()) + .is_err() + ); + assert!(compute_cells(&invalid[1..]).is_err()); + } + + /// [`verify_cell_kzg_proof_batch`] gives c-kzg's answer, `Ok(true)`, + /// `Ok(false)` or `Err`, on valid batches of several shapes and on every + /// way a batch can be wrong or malformed. + #[test] + fn verify_cell_kzg_proof_batch_agrees_with_c_kzg() { + let blobs: Vec> = (0..3).map(|seed| test_blob(100 + seed)).collect(); + let mut commitments = Vec::new(); + let mut all_cells = Vec::new(); + let mut all_proofs = Vec::new(); + for blob in &blobs { + commitments.push(blob_to_kzg_commitment(blob).unwrap()); + let (cells, proofs) = compute_cells_and_kzg_proofs(blob).unwrap(); + all_cells.push(cells); + all_proofs.push(proofs); + } + let batch = |entries: &[(usize, usize)]| -> Batch { + let mut batch: Batch = Default::default(); + for &(blob, cell) in entries { + batch.0.push(commitments[blob]); + batch.1.push(cell as u64); + batch.2.push(all_cells[blob][cell]); + batch.3.push(all_proofs[blob][cell]); + } + batch + }; + let every_cell: Vec<(usize, usize)> = (0..blobs.len()) + .flat_map(|blob| (0..CELLS_PER_EXT_BLOB).map(move |cell| (blob, cell))) + .collect(); + let one_column: Vec<(usize, usize)> = (0..blobs.len()).map(|blob| (blob, 5)).collect(); + let a_few: Vec<(usize, usize)> = vec![(0, 0), (2, 127), (1, 64), (0, 3), (0, 3)]; + + let mut cases: Vec<(&str, Batch)> = vec![ + ("every cell", batch(&every_cell)), + ("one column", batch(&one_column)), + ("a few, one repeated", batch(&a_few)), + ("empty", batch(&[])), + ]; + let base = batch(&one_column); + let mut changed = |name: &'static str, edit: &dyn Fn(&mut Batch)| { + let mut batch = base.clone(); + edit(&mut batch); + cases.push((name, batch)); + }; + changed("a cell's element changed, still canonical", &|b| { + b.2[1][31] ^= 1 + }); + changed("two proofs swapped", &|b| b.3.swap(0, 2)); + changed("a cell index changed", &|b| b.1[0] = 6); + changed("a commitment swapped", &|b| b.0.swap(0, 1)); + changed("a non-canonical element", &|b| b.2[2][0] = 0xff); + changed("a proof that does not decode", &|b| b.3[1].0[0] ^= 0x80); + changed("a commitment that does not decode", &|b| { + b.0[0].0[0] ^= 0x80 + }); + changed("the infinity commitment", &|b| { + b.0[0].0 = [0; COMPRESSED_LEN]; + b.0[0].0[0] = 0xc0; + }); + changed("a cell index past the extended blob", &|b| { + b.1[0] = CELLS_PER_EXT_BLOB as u64; + }); + changed("one cell short", &|b| { + b.2.pop(); + }); + changed("one proof short", &|b| { + b.3.pop(); + }); + + for (name, (commitments, indices, cells, proofs)) in &cases { + let ours = + verify_cell_kzg_proof_batch(commitments, indices, cells, proofs).map_err(|_| ()); + let theirs = c_kzg_verdict(commitments, indices, cells, proofs); + assert_eq!(ours, theirs, "{name}"); + } + // The valid shapes do verify, so the comparison above is not vacuous. + assert_eq!(c_kzg_verdict(&base.0, &base.1, &base.2, &base.3), Ok(true)); + } } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/kzg.rs b/crates/blockchain/state_transition/tests/beacon_spec/kzg.rs index b3aa9b43a..2c38c689a 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/kzg.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/kzg.rs @@ -57,9 +57,8 @@ //! to the match would be dead weight, not a safety net, since no arm could //! ever need it. -use c_kzg::Cell; use ethlambda_state_transition::beacon::ForkName; -use ethlambda_state_transition::beacon::kzg; +use ethlambda_state_transition::beacon::kzg::{self, Cell}; use ethlambda_state_transition::beacon::primitives::{Bytes32, H256, KzgCommitment, KzgProof}; use libtest_mimic::Trial; use serde_yaml_ng::Value; @@ -108,13 +107,9 @@ fn hex_proof(value: &Value) -> Option { hex_array(value).map(KzgProof) } -/// Decodes one cell of an extended blob. -/// -/// Unlike the fixed arrays above, [`Cell::from_bytes`] does its own length -/// check rather than this runner doing it through `try_into`, since `Cell` is -/// `c-kzg`'s type, not this crate's; `None` on a length mismatch either way. +/// Decodes one cell of an extended blob, a fixed array like the fields above. fn hex_cell(value: &Value) -> Option { - Cell::from_bytes(&hex_bytes(value)).ok() + hex_array(value) } /// Parses a YAML list of plain integers (`cell_indices`, `commitment_indices`), diff --git a/crates/common/crypto/Cargo.toml b/crates/common/crypto/Cargo.toml index c324d55e1..125a468bf 100644 --- a/crates/common/crypto/Cargo.toml +++ b/crates/common/crypto/Cargo.toml @@ -11,6 +11,10 @@ version.workspace = true [dependencies] blst = "0.3.16" +# Point decoding, subgroup checks and pairing products, eight lanes at a time on +# AVX-512 IFMA CPUs and through blst elsewhere; see `bls`'s module documentation +# for which operations still go to blst directly. +silver_crypto_simd.workspace = true ethlambda-metrics.workspace = true ethlambda-types.workspace = true rand.workspace = true diff --git a/crates/common/crypto/src/bls.rs b/crates/common/crypto/src/bls.rs index ae57ef6d0..40776f499 100644 --- a/crates/common/crypto/src/bls.rs +++ b/crates/common/crypto/src/bls.rs @@ -1,12 +1,31 @@ //! The specification's `bls` module: `bls.Verify`, `bls.Aggregate`, and friends. //! -//! The state transition never touches `blst` directly; it calls the functions -//! here, named exactly as the spec names them (`specs/phase0/beacon-chain.md`'s -//! "BLS signatures" section, extended by `specs/altair/bls.md`), so a reader can -//! match a call site against the spec line by line. `blst` is the only backend: -//! there is no trait to abstract over another one, since the consensus layer has -//! settled on `blst` as the reference implementation and a second backend would -//! only be dead code here. +//! The state transition never touches a BLS library directly; it calls the +//! functions here, named exactly as the spec names them +//! (`specs/phase0/beacon-chain.md`'s "BLS signatures" section, extended by +//! `specs/altair/bls.md`), so a reader can match a call site against the spec +//! line by line. +//! +//! # Which library does what +//! +//! Two libraries sit behind these functions, split by operation rather than +//! behind a trait, since each operation has exactly one implementation: +//! +//! - [`silver_crypto_simd`] decodes and subgroup-checks every point (public +//! keys and signatures) and evaluates every pairing equation. On an x86-64 +//! CPU with AVX-512 IFMA it works on eight points or eight Miller loops at a +//! time; elsewhere it falls back to `blst` and portable Rust through the same +//! API. Its decoders return `blst`'s own verdict on every input, and its +//! pairing product checks the same equation `blst`'s pairing context does. +//! - `blst` still sums points (key aggregates, committee sums, signature +//! aggregates), signs, and generates keys: `silver_crypto_simd` offers no +//! faster route for those, and its points convert to and from `blst`'s +//! affine form without a check. +//! +//! The one behavior the two disagree on is the identity in a pairing: +//! `silver_crypto_simd`'s product drops a pair holding it, where `blst` refused +//! an identity public key. [`core_verify`] and the batch check refuse that key +//! themselves before any pairing runs; see there. //! //! # Why every function treats its inputs as unvalidated //! @@ -34,7 +53,7 @@ //! key validation, and because every active validator's key signs again each //! epoch, almost all of it was repeated work. //! -//! So [`PublicKey::key_validate`]'s answer is memoized in [`PubkeyCache`], +//! So the answer to `KeyValidate` is memoized in [`PubkeyCache`], //! which holds it in two structures. Both are keyed, in the end, by the exact //! compressed bytes the answer was asked about: //! @@ -108,26 +127,30 @@ //! `crates/blockchain/src/lib.rs`), so on a multi-core host every one of those //! checks but the one currently running would leave a core idle. Validating one //! key never reads or writes anything another key's validation touches, so -//! nothing depends on which order they run in or finish in: the misses go -//! through `par_iter().map(...).collect::>>()`, which spreads the -//! checks across rayon's global thread pool and still collapses to `None` the -//! moment any key fails, without skipping or weakening [`key_validate`] for a -//! single key. Each validated point is written back to its own index, which -//! `aggregate_verify` relies on to keep `points[i]` paired with `messages[i]`. +//! nothing depends on which order they run in or finish in: the misses are cut +//! into chunks of [`KEY_DECODE_CHUNK`] keys, each chunk goes to +//! `silver_crypto_simd`'s batch decoder (eight keys per step on an IFMA CPU) on +//! rayon's global thread pool, and the result collapses to `None` if any key +//! fails, without skipping or weakening [`key_validate`] for a single key. Each +//! validated point is written back to its own index, which `aggregate_verify` +//! relies on to keep `points[i]` paired with `messages[i]`. //! //! A hit is a hash-map read, far cheaper than handing work to another thread, -//! so hits resolve on the calling thread. That matters beyond the saving -//! itself: gossip validation runs many signature checks at once on blocking -//! threads, and if every one of them queued its keys on rayon's single global -//! pool, a burst of aggregates would stall there even with the keys already -//! validated. +//! so hits resolve on the calling thread, and so does a miss set no larger than +//! one chunk. That matters beyond the saving itself: gossip validation runs many +//! signature checks at once on blocking threads, and if every one of them +//! queued its keys on rayon's single global pool, a burst of aggregates would +//! stall there even with the keys already validated. use std::collections::HashMap; use std::sync::{LazyLock, OnceLock, RwLock}; +use blst::MultiPoint; use blst::min_pk::{AggregatePublicKey, AggregateSignature, PublicKey, Signature}; -use blst::{BLST_ERROR, MultiPoint}; use rayon::prelude::*; +use silver_crypto_simd::bls::{ + G1Point, G2Point, PairingProduct, blst_p1_affine, uncompress_in_g1, uncompress_in_g2, +}; use ethlambda_types::beacon::error::{Error, Result}; use ethlambda_types::beacon::primitives::{ @@ -148,21 +171,18 @@ pub use batch::{SignatureSet, verify_signature_sets}; /// signature reuse. pub const DST: &[u8] = b"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_"; -/// The minimum number of public keys rayon may put in one sequential chunk -/// when [`aggregate_verify`] and [`fast_aggregate_verify`] validate the -/// signers [`PubkeyCache`] has not seen yet in parallel. +/// How many public keys [`PubkeyCache`] has not seen go to one call of +/// `silver_crypto_simd`'s batch decoder when [`aggregate_verify`] and +/// [`fast_aggregate_verify`] validate them in parallel. /// -/// A devnet or a spec-fixture aggregate can be as small as a single signer, -/// and splitting a handful of keys across worker threads would spend more -/// time on task dispatch than [`PublicKey::key_validate`] itself takes to -/// decompress and subgroup-check one key. `with_min_len` keeps that decision -/// inside rayon's own splitting logic rather than a hand-rolled length check -/// before choosing serial or parallel: sixteen is comfortably above the -/// single-digit-signer inputs a fixture or a small devnet produces, so those -/// stay on the calling thread, while a mainnet aggregate of hundreds to -/// thousands of signers still splits into far more chunks than there are -/// cores to run them on. -const KEY_VALIDATE_MIN_PAR_LEN: usize = 16; +/// On an IFMA CPU the decoder works on eight keys per step and finishes a +/// partial step one key at a time, so a chunk is a multiple of eight. A chunk +/// is also the unit rayon schedules: this size keeps a task well above rayon's +/// dispatch cost, while a mainnet block's first import, with tens of thousands +/// of keys not yet seen, still splits into far more chunks than the pool has +/// threads. A miss set no larger than one chunk (a fixture's, a devnet's, most +/// gossip's) is decoded on the calling thread. +const KEY_DECODE_CHUNK: usize = 64; /// How many shards [`PubkeyCache`] splits its map across. /// @@ -184,7 +204,7 @@ const PUBKEY_CACHE_SHARDS: usize = 64; /// cache existed. const MAX_CACHED_PUBKEYS: usize = 1 << 22; -/// Every public key that has passed [`PublicKey::key_validate`], by its +/// Every public key that has passed `KeyValidate`, by its /// compressed bytes, for the whole process. /// /// Process-wide rather than owned by a `Store`, because nothing about the @@ -273,7 +293,7 @@ pub struct IndexedPubkey { /// Validated, decompressed public keys, by their compressed encoding or by the /// registry index they were read at. /// -/// A bounded, append-only memo of [`PublicKey::key_validate`]: an entry is +/// A bounded, append-only memo of `KeyValidate`: an entry is /// only ever added for a key that passed, never changed, and never removed, /// since its answer can never change. See the module documentation for why /// that is sound. @@ -420,8 +440,87 @@ impl PubkeyCache { } } +/// A decoded G1 point as a public key, or `None` for the identity. +/// +/// `silver_crypto_simd`'s decoder admits the identity, since it is a point of +/// the subgroup; `KeyValidate` refuses it, as `blst`'s `key_validate` did. +fn admit_pubkey(point: G1Point) -> Option { + (!point.is_infinity()).then(|| PublicKey::from(point.into_blst())) +} + +/// `KeyValidate` on one encoding, uncached: the point, or `None` if the bytes +/// do not decode to a subgroup point of G1 or decode to the identity. +fn decode_pubkey(encoding: &[u8; BLS_PUBKEY_SIZE]) -> Option { + let mut decoded = [None]; + uncompress_in_g1(std::slice::from_ref(encoding), &mut decoded); + decoded[0].and_then(admit_pubkey) +} + +/// `KeyValidate` on each encoding, uncached and in order. +/// +/// Chunks of [`KEY_DECODE_CHUNK`] go to rayon's pool, unless there is only one; +/// see the module documentation. +fn decode_pubkeys(encodings: &[[u8; BLS_PUBKEY_SIZE]]) -> Vec> { + let mut decoded = vec![None; encodings.len()]; + if encodings.len() <= KEY_DECODE_CHUNK { + uncompress_in_g1(encodings, &mut decoded); + } else { + encodings + .par_chunks(KEY_DECODE_CHUNK) + .zip(decoded.par_chunks_mut(KEY_DECODE_CHUNK)) + .for_each(|(input, output)| uncompress_in_g1(input, output)); + } + decoded + .into_iter() + .map(|point| point.and_then(admit_pubkey)) + .collect() +} + +/// `signature` as a subgroup point of G2, or `None` if it does not decode to +/// one. The identity decodes: it is a valid signature encoding (the one an +/// empty aggregate carries), which `blst`'s `sig_validate(_, false)` accepted +/// too. +fn decode_signature(signature: &BlsSignature) -> Option { + let mut decoded = [None]; + uncompress_in_g2(std::slice::from_ref(&signature.0), &mut decoded); + decoded[0] +} + +/// A key this module validated, as `silver_crypto_simd`'s point type. +/// +/// Every [`PublicKey`] this module holds passed `KeyValidate` or is a sum of +/// keys that did, so it is a canonical point of G1's prime-order subgroup, +/// which is the whole of `from_trusted`'s contract. +fn g1_point(pubkey: &PublicKey) -> G1Point { + G1Point::from_trusted(blst_p1_affine::from(*pubkey)) +} + +/// The pairing equation every single-message check comes down to, +/// `e(pubkey, H(message)) == e(G1, signature)`, as one product of two +/// pairings. +/// +/// The identity key is refused before the pairing. `silver_crypto_simd`'s +/// product drops a pair holding the identity, so with that key the product +/// would reduce to `e(-G1, signature)`, which the identity signature +/// satisfies: a set of keys that cancel (`P` and `-P`, both valid keys) would +/// then verify under no signature at all. `blst` refused such a key at pairing +/// time (`BLST_PK_IS_INFINITY`), and IETF `CoreVerify` runs `KeyValidate` on +/// the key it is given, which for `FastAggregateVerify` is the sum. Only a sum +/// can be the identity: a single key is refused by `KeyValidate` first. +fn core_verify(pubkey: &PublicKey, message: Root, signature: &G2Point) -> bool { + let pubkey = g1_point(pubkey); + if pubkey.is_infinity() { + return false; + } + let hash = G2Point::hash_to_curve(message.as_slice(), DST); + PairingProduct::two_pairs_are_one( + (&G1Point::negative_generator(), signature), + (&pubkey, &hash), + ) +} + /// `pubkey` as a validated point, or `None` if it is not a valid, -/// subgroup-correct, non-identity key: [`PublicKey::key_validate`], through +/// subgroup-correct, non-identity key: `KeyValidate`, through /// [`VALIDATED_PUBKEYS`]. fn validated_pubkey(pubkey: &K) -> Option { if let Some(point) = VALIDATED_PUBKEYS.get(pubkey) { @@ -429,7 +528,7 @@ fn validated_pubkey(pubkey: &K) -> Option { return Some(point); } crate::metrics::inc_pubkey_cache_lookups(0, 1); - let point = PublicKey::key_validate(pubkey.pubkey().as_ref()).ok()?; + let point = decode_pubkey(&pubkey.pubkey().0)?; VALIDATED_PUBKEYS.insert(pubkey, point); Some(point) } @@ -454,10 +553,12 @@ fn validated_pubkeys(pubkeys: &[K]) -> Option> { crate::metrics::inc_pubkey_cache_lookups(hits as u64, misses.len() as u64); if !misses.is_empty() { - let fresh: Vec = misses - .par_iter() - .with_min_len(KEY_VALIDATE_MIN_PAR_LEN) - .map(|&index| PublicKey::key_validate(pubkeys[index].pubkey().as_ref()).ok()) + let encodings: Vec<[u8; BLS_PUBKEY_SIZE]> = misses + .iter() + .map(|&index| pubkeys[index].pubkey().0) + .collect(); + let fresh: Vec = decode_pubkeys(&encodings) + .into_iter() .collect::>()?; for (&index, point) in misses.iter().zip(fresh) { VALIDATED_PUBKEYS.insert(&pubkeys[index], point); @@ -469,7 +570,7 @@ fn validated_pubkeys(pubkeys: &[K]) -> Option> { /// The sum of `points` as a projective point, or `None` for an empty slice. /// -/// Every point must already have passed [`PublicKey::key_validate`] (they come +/// Every point must already have passed `KeyValidate` (they come /// from [`validated_pubkeys`]), so the sum skips re-checking them. It goes /// through blst's `MultiPoint::add`, which hands the whole slice to /// `blst_p1s_add`: a batched affine addition that shares one field inversion @@ -543,11 +644,10 @@ pub fn verify(pubkey: &K, message: Root, signature: &BlsSignatu let Some(pubkey) = validated_pubkey(pubkey) else { return false; }; - let Ok(signature) = Signature::sig_validate(signature.as_ref(), false) else { + let Some(signature) = decode_signature(signature) else { return false; }; - let result = signature.verify(false, message.as_slice(), DST, &[], &pubkey, false); - result == BLST_ERROR::BLST_SUCCESS + core_verify(&pubkey, message, &signature) } /// The specification's `bls.Aggregate(signatures) -> BLSSignature`. @@ -559,14 +659,24 @@ pub fn verify(pubkey: &K, message: Root, signature: &BlsSignatu /// curve addition of two points outside the prime-order subgroup can still /// land back inside it: an invalid share could otherwise cancel against /// another invalid share and slip past a check performed only on the result. +/// +/// The shares are decoded together, so an IFMA CPU checks eight per step; the +/// sum is `blst`'s. pub fn aggregate(signatures: &[BlsSignature]) -> Result { ethlambda_types::beacon::error::verify(!signatures.is_empty(), "len(signatures) > 0")?; - let encoded: Vec<&[u8]> = signatures - .iter() - .map(|signature| signature.as_ref()) - .collect(); - let aggregated = AggregateSignature::aggregate_serialized(&encoded, true) - .map_err(|_| Error::InvalidSignature("aggregate: not a valid, subgroup-correct point"))?; + let invalid = || Error::InvalidSignature("aggregate: not a valid, subgroup-correct point"); + let encodings: Vec<[u8; BLS_SIGNATURE_SIZE]> = + signatures.iter().map(|signature| signature.0).collect(); + let mut decoded = vec![None; encodings.len()]; + uncompress_in_g2(&encodings, &mut decoded); + let shares: Vec = decoded + .into_iter() + .map(|point| point.map(|point| Signature::from(point.into_blst()))) + .collect::>() + .ok_or_else(invalid)?; + let share_refs: Vec<&Signature> = shares.iter().collect(); + // Every share was decoded and subgroup-checked above. + let aggregated = AggregateSignature::aggregate(&share_refs, false).map_err(|_| invalid())?; Ok(BlsSignature(aggregated.to_signature().to_bytes())) } @@ -590,16 +700,21 @@ pub fn aggregate_verify( if pubkeys.is_empty() || pubkeys.len() != messages.len() { return false; } - let Ok(signature) = Signature::sig_validate(signature.as_ref(), false) else { + let Some(signature) = decode_signature(signature) else { return false; }; let Some(points) = validated_pubkeys(pubkeys) else { return false; }; - let point_refs: Vec<&PublicKey> = points.iter().collect(); - let message_refs: Vec<&[u8]> = messages.iter().map(Root::as_slice).collect(); - let result = signature.aggregate_verify(false, &message_refs, DST, &point_refs, false); - result == BLST_ERROR::BLST_SUCCESS + // `prod e(pk_i, H(m_i)) == e(G1, signature)`. Every key passed + // `KeyValidate`, so none is the identity and no pair is dropped for one. + let mut product = PairingProduct::with_capacity(points.len() + 1); + product.push(&G1Point::negative_generator(), &signature); + for (point, message) in points.iter().zip(messages) { + let hash = G2Point::hash_to_curve(message.as_slice(), DST); + product.push(&g1_point(point), &hash); + } + product.is_one() } /// The specification's @@ -621,20 +736,18 @@ pub fn fast_aggregate_verify( if pubkeys.is_empty() { return false; } - let Ok(signature) = Signature::sig_validate(signature.as_ref(), false) else { + let Some(signature) = decode_signature(signature) else { return false; }; let Some(points) = validated_pubkeys(pubkeys) else { return false; }; - // `pubkeys` is non-empty, so there is always a sum. The signature was - // validated above, which is why the group check stays off here. + // `pubkeys` is non-empty, so there is always a sum. It may be the + // identity, which `core_verify` refuses. let Some(aggregated) = sum_validated_points(&points) else { return false; }; - let result = - signature.fast_aggregate_verify_pre_aggregated(false, message.as_slice(), DST, &aggregated); - result == BLST_ERROR::BLST_SUCCESS + core_verify(&aggregated, message, &signature) } /// `specs/altair/bls.md`'s `eth_aggregate_pubkeys(pubkeys) -> BLSPubkey`. @@ -774,9 +887,11 @@ mod tests { fn mixed_hits_and_misses_keep_their_positions() { let warm = [fresh_pubkey(1), fresh_pubkey(2)]; assert!(warm.iter().all(key_validate)); - // Misses between hits, in a run long enough to go through rayon. + // Misses between hits, in a run long enough to go through rayon and to + // end on a partial chunk. + let misses = 2 * KEY_DECODE_CHUNK as u64 + 3; let mut pubkeys = vec![warm[0]]; - pubkeys.extend((10..10 + 2 * KEY_VALIDATE_MIN_PAR_LEN as u64).map(fresh_pubkey)); + pubkeys.extend((10..10 + misses).map(fresh_pubkey)); pubkeys.push(warm[1]); let points = validated_pubkeys(&pubkeys).expect("every key above is valid"); @@ -1250,4 +1365,225 @@ mod tests { } } } + + /// Encodings `KeyValidate` must judge, with every way to fail: valid keys, + /// the same keys with each flag bit flipped (no compression flag, a + /// spurious infinity flag, the other `y`), the identity and malformed + /// identities, an `x` at or past the field modulus, and random `x` + /// coordinates, about half of which are on the curve and almost none of + /// those in the subgroup. + fn key_encodings_to_judge() -> Vec<[u8; BLS_PUBKEY_SIZE]> { + use rand::RngExt; + let mut rng = rand::rng(); + let mut encodings = Vec::new(); + for seed in 0..40 { + let valid = fresh_pubkey(5000 + seed).0; + encodings.push(valid); + for flag in [0x80, 0x40, 0x20] { + let mut flipped = valid; + flipped[0] ^= flag; + encodings.push(flipped); + } + let mut last = valid; + last[BLS_PUBKEY_SIZE - 1] ^= 1; + encodings.push(last); + } + encodings.push(identity_pubkey().0); + let mut noisy_identity = identity_pubkey().0; + noisy_identity[BLS_PUBKEY_SIZE - 1] = 1; + encodings.push(noisy_identity); + encodings.push([0; BLS_PUBKEY_SIZE]); + encodings.push([0xff; BLS_PUBKEY_SIZE]); + let mut past_modulus = [0xff; BLS_PUBKEY_SIZE]; + past_modulus[0] = 0x9f; + encodings.push(past_modulus); + for _ in 0..200 { + let mut random = [0u8; BLS_PUBKEY_SIZE]; + rng.fill(&mut random[..]); + random[0] = (random[0] & 0x1f) | 0x80; + encodings.push(random); + } + encodings + } + + /// The batch decoder, with its chunking and (on an IFMA CPU) its eight-lane + /// kernel, gives `blst`'s `key_validate` verdict on every encoding, and so + /// does the single-key path. + #[test] + fn key_decoding_agrees_with_blst_on_valid_and_corrupt_encodings() { + let encodings = key_encodings_to_judge(); + assert!(encodings.len() > 2 * KEY_DECODE_CHUNK); + let reference: Vec> = encodings + .iter() + .map(|bytes| PublicKey::key_validate(bytes).ok().map(|p| p.to_bytes())) + .collect(); + assert!(reference.iter().any(Option::is_some)); + assert!(reference.iter().filter(|r| r.is_none()).count() > encodings.len() / 2); + + let batch = decode_pubkeys(&encodings); + for (index, bytes) in encodings.iter().enumerate() { + let batched = batch[index].map(|p| p.to_bytes()); + let single = decode_pubkey(bytes).map(|p| p.to_bytes()); + assert_eq!( + batched, reference[index], + "batch, encoding {index}: {bytes:02x?}" + ); + assert_eq!( + single, reference[index], + "single, encoding {index}: {bytes:02x?}" + ); + } + } + + /// The G2 decoder gives `blst`'s `sig_validate(_, false)` verdict on valid + /// signatures, corrupted ones, the identity and random `x` coordinates, + /// alone and in batches long enough for the eight-lane kernel. + #[test] + fn signature_decoding_agrees_with_blst_on_valid_and_corrupt_encodings() { + use rand::RngExt; + let mut rng = rand::rng(); + let (_, _, aggregate) = build_aggregate(3); + let mut encodings = Vec::new(); + for seed in 0..24u8 { + let ikm = [seed.wrapping_add(7); 32]; + let secret = blst::min_pk::SecretKey::key_gen(&ikm, &[]).unwrap(); + let valid = secret.sign(&[seed; 32], DST, &[]).to_bytes(); + encodings.push(valid); + for flag in [0x80, 0x40, 0x20] { + let mut flipped = valid; + flipped[0] ^= flag; + encodings.push(flipped); + } + let mut last = valid; + last[BLS_SIGNATURE_SIZE - 1] ^= 1; + encodings.push(last); + } + encodings.push(aggregate.0); + encodings.push(G2_POINT_AT_INFINITY); + let mut noisy_infinity = G2_POINT_AT_INFINITY; + noisy_infinity[BLS_SIGNATURE_SIZE - 1] = 1; + encodings.push(noisy_infinity); + encodings.push([0; BLS_SIGNATURE_SIZE]); + for _ in 0..64 { + let mut random = [0u8; BLS_SIGNATURE_SIZE]; + rng.fill(&mut random[..]); + random[0] = (random[0] & 0x1f) | 0x80; + encodings.push(random); + } + let reference: Vec> = encodings + .iter() + .map(|bytes| { + Signature::sig_validate(bytes, false) + .ok() + .map(|s| s.to_bytes()) + }) + .collect(); + assert!(reference.iter().any(Option::is_some)); + assert!(reference.iter().any(Option::is_none)); + + let to_bytes = |point: G2Point| Signature::from(point.into_blst()).to_bytes(); + let mut batch = vec![None; encodings.len()]; + uncompress_in_g2(&encodings, &mut batch); + for (index, bytes) in encodings.iter().enumerate() { + let single = decode_signature(&BlsSignature(*bytes)).map(to_bytes); + let batched = batch[index].map(to_bytes); + assert_eq!(batched, reference[index], "batch, encoding {index}"); + assert_eq!(single, reference[index], "single, encoding {index}"); + } + } + + /// `n` keypairs, distinct per `(tag, n)`, as secrets and as encodings. + fn keypairs(tag: u64, n: usize) -> (Vec, Vec) { + let secrets: Vec = (0..n as u64) + .map(|i| { + let mut ikm = [0x3cu8; 32]; + ikm[..8].copy_from_slice(&(tag * 1000 + i).to_le_bytes()); + blst::min_pk::SecretKey::key_gen(&ikm, &[]).unwrap() + }) + .collect(); + let pubkeys = secrets + .iter() + .map(|secret| BlsPubkey(secret.sk_to_pk().to_bytes())) + .collect(); + (secrets, pubkeys) + } + + fn sign(secret: &blst::min_pk::SecretKey, message: Root) -> BlsSignature { + BlsSignature(secret.sign(message.as_slice(), DST, &[]).to_bytes()) + } + + /// `verify`, `fast_aggregate_verify` and `aggregate_verify` give the answer + /// `blst`'s own checks give, over valid inputs, wrong messages, short + /// aggregates, the identity signature, and signer counts on both sides of + /// the pairing kernel's eight lanes. + #[test] + fn verification_agrees_with_blst() { + use blst::BLST_ERROR::BLST_SUCCESS; + use ethlambda_types::beacon::primitives::H256; + + let message = H256([0x51; 32]); + let other = H256([0x52; 32]); + let infinity = BlsSignature(G2_POINT_AT_INFINITY); + let blst_sig = |s: &BlsSignature| Signature::sig_validate(s.as_ref(), false).unwrap(); + for count in [1usize, 2, 7, 8, 9, 17] { + let (secrets, pubkeys) = keypairs(count as u64, count); + let blst_keys: Vec = secrets.iter().map(|s| s.sk_to_pk()).collect(); + let key_refs: Vec<&PublicKey> = blst_keys.iter().collect(); + let shares: Vec = secrets.iter().map(|s| sign(s, message)).collect(); + let good = aggregate(&shares).unwrap(); + let short = match count { + 1 => infinity, + _ => aggregate(&shares[1..]).unwrap(), + }; + + let cases = [ + (good, message), + (good, other), + (short, message), + (infinity, message), + ]; + for (signature, msg) in cases { + let theirs = blst_sig(&signature).fast_aggregate_verify( + false, + msg.as_slice(), + DST, + &key_refs, + ) == BLST_SUCCESS; + let ours = fast_aggregate_verify(&pubkeys, msg, &signature); + assert_eq!(ours, theirs, "fast_aggregate_verify, {count} signers"); + } + for ((pubkey, secret), point) in pubkeys.iter().zip(&secrets).zip(&blst_keys) { + let own = sign(secret, message); + for (signature, msg) in [(own, message), (own, other), (infinity, message)] { + let theirs = + blst_sig(&signature).verify(false, msg.as_slice(), DST, &[], point, false) + == BLST_SUCCESS; + assert_eq!(verify(pubkey, msg, &signature), theirs, "verify"); + } + } + + // Distinct messages, one per signer, and the same messages rotated. + let messages: Vec = (0..count).map(|i| H256([i as u8; 32])).collect(); + let distinct: Vec = secrets + .iter() + .zip(&messages) + .map(|(secret, msg)| sign(secret, *msg)) + .collect(); + let combined = aggregate(&distinct).unwrap(); + let mut rotated = messages.clone(); + rotated.rotate_left(1); + for msgs in [&messages, &rotated] { + let message_refs: Vec<&[u8]> = msgs.iter().map(Root::as_slice).collect(); + let theirs = blst_sig(&combined).aggregate_verify( + false, + &message_refs, + DST, + &key_refs, + false, + ) == BLST_SUCCESS; + let ours = aggregate_verify(&pubkeys, msgs, &combined); + assert_eq!(ours, theirs, "aggregate_verify, {count} signers"); + } + } + } } diff --git a/crates/common/crypto/src/bls/batch.rs b/crates/common/crypto/src/bls/batch.rs index adda00f62..cb73e6647 100644 --- a/crates/common/crypto/src/bls/batch.rs +++ b/crates/common/crypto/src/bls/batch.rs @@ -9,7 +9,8 @@ //! in full. Checking `n` equations `e(pk_i, H(m_i)) = e(g1, sig_i)` together //! needs one Miller loop per set but only one final exponentiation for all of //! them, so a block with a dozen signature sets spends a fraction of what a -//! dozen separate checks do. +//! dozen separate checks do. `silver_crypto_simd` runs those Miller loops eight +//! at a time on an IFMA CPU, which a batch gives it enough of to fill. //! //! # Why the weights are random //! @@ -26,6 +27,23 @@ //! operating system's entropy and reseeded periodically, and a weight of zero //! is redrawn, since it would drop its set from the equation altogether. //! +//! The combined equation is `e(-g1, sum(r_i * sig_i)) * prod e(r_i * pk_i, +//! H(m_i)) == 1`: the weights go on the signatures through one multi-scalar +//! sum, and on each key through its own scalar multiplication, which in G1 is +//! the cheaper group to multiply in. +//! +//! # Why a batch is split into chunks +//! +//! Hashing each message to G2 and weighting its key cost more than the +//! pairing itself, and blst's batch, which this replaced, spread them over +//! its thread pool. A batch here is cut into chunks of [`SETS_PER_CHUNK`] +//! sets, each its own random linear combination with its own weights and its +//! own final exponentiation, checked in parallel on rayon's pool with each +//! chunk's hashes in parallel too. A chunk plus the pair for its signature sum +//! fills one pass of the Miller-loop lanes. Splitting keeps the soundness +//! argument per chunk: a forged set sits in one chunk, and that chunk passes +//! with probability about 2^-64 whatever the other chunks hold. +//! //! # What a batch answers //! //! Only whether every set holds. When it fails it cannot say which set did; @@ -34,21 +52,18 @@ //! [`super::verify`] or [`super::fast_aggregate_verify`] gives for the same //! inputs. -use blst::BLST_ERROR; -use blst::blst_scalar; -use blst::min_pk::{PublicKey, Signature}; +use blst::min_pk::PublicKey; use rand::RngExt; +use rayon::prelude::*; +use silver_crypto_simd::bls::limbs::LANES; +use silver_crypto_simd::bls::{G1Point, G2Point, PairingProduct}; use ethlambda_types::beacon::primitives::{BlsSignature, Root}; -use super::{AggregatePubkey, DST, PubkeySource, validated_pubkey, validated_pubkeys}; - -/// How many bits of each random weight the batch check uses. -const RAND_BITS: usize = 64; - -/// The bit of a compressed G1 point's first byte that marks the point at -/// infinity. -const INFINITY_FLAG: u8 = 0b0100_0000; +use super::{ + AggregatePubkey, DST, PubkeySource, core_verify, decode_signature, g1_point, validated_pubkey, + validated_pubkeys, +}; /// One signature check, resolved and ready, that may be verified alone or in a /// batch. @@ -58,14 +73,15 @@ const INFINITY_FLAG: u8 = 0b0100_0000; /// everything a check does short of the pairing. That split is what lets a /// caller decide *when* to pay for the pairing: at once ([`Self::verify`]) or /// together with the other sets of a block ([`verify_signature_sets`]). +/// +/// A single key and a sum of keys are checked by the same equation, so a set +/// does not record which one it holds; only a sum can be the identity, which +/// both checks refuse. #[derive(Clone, Copy)] pub struct SignatureSet { pubkey: PublicKey, message: Root, - signature: Signature, - /// Whether `pubkey` is a sum of keys, checked the way - /// `bls.FastAggregateVerify` checks one, rather than a single registry key. - aggregated: bool, + signature: G2Point, } impl SignatureSet { @@ -79,19 +95,18 @@ impl SignatureSet { signature: &BlsSignature, ) -> Option { let pubkey = validated_pubkey(pubkey)?; - let signature = Signature::sig_validate(signature.as_ref(), false).ok()?; + let signature = decode_signature(signature)?; Some(Self { pubkey, message, signature, - aggregated: false, }) } /// `bls.FastAggregateVerify`'s inputs. /// - /// `None` for an empty `pubkeys`, any key that fails `key_validate`, or a - /// signature that fails `sig_validate`: the cases + /// `None` for an empty `pubkeys`, any key that fails `KeyValidate`, or a + /// signature that does not decode to a subgroup point: the cases /// [`super::fast_aggregate_verify`] answers false for before the pairing. pub fn fast_aggregate( pubkeys: &[K], @@ -101,97 +116,75 @@ impl SignatureSet { if pubkeys.is_empty() { return None; } - let signature = Signature::sig_validate(signature.as_ref(), false).ok()?; + let signature = decode_signature(signature)?; let points = validated_pubkeys(pubkeys)?; - // Every point above already passed `key_validate`, so the sum does + // Every point above already passed `KeyValidate`, so the sum does // not re-check them. let pubkey = super::sum_validated_points(&points)?; Some(Self { pubkey, message, signature, - aggregated: true, }) } /// `bls.FastAggregateVerify`'s inputs with the key already summed, as /// committee key sums produce it. /// - /// `None` when `signature` fails `sig_validate`. The key needs no check: a - /// sum of validated points is in the subgroup, and the identity, the one - /// sum that is not a valid key, is refused by [`Self::verify`] and by the - /// batch at pairing time, as it is for a key summed here from scratch. + /// `None` when `signature` does not decode to a subgroup point. The key + /// needs no check: a sum of validated points is in the subgroup, and the + /// identity, the one sum that is not a valid key, is refused by + /// [`Self::verify`] and by the batch before any pairing, as it is for a + /// key summed here from scratch. pub fn from_aggregate( key: AggregatePubkey, message: Root, signature: &BlsSignature, ) -> Option { - let signature = Signature::sig_validate(signature.as_ref(), false).ok()?; + let signature = decode_signature(signature)?; Some(Self { pubkey: key.to_public_key(), message, signature, - aggregated: true, }) } /// The pairing check on its own: the same answer [`super::verify`] or /// [`super::fast_aggregate_verify`] gives for these inputs. pub fn verify(&self) -> bool { - let result = if self.aggregated { - self.signature.fast_aggregate_verify_pre_aggregated( - false, - self.message.as_slice(), - DST, - &self.pubkey, - ) - } else { - self.signature.verify( - false, - self.message.as_slice(), - DST, - &[], - &self.pubkey, - false, - ) - }; - result == BLST_ERROR::BLST_SUCCESS + core_verify(&self.pubkey, self.message, &self.signature) } /// Whether the key is the identity, which no signature can validly be /// made under. Only a sum of keys can be: a single registry key is - /// rejected by `key_validate` if it is. + /// rejected by `KeyValidate` if it is. /// - /// blst rejects the same input itself (`BLST_PK_IS_INFINITY`, from the - /// pairing aggregation both the single and the batch check go through), - /// so this makes the existing rule explicit and changes no behavior. + /// `silver_crypto_simd`'s pairing product would drop the pair, so this + /// check is what refuses the set; see `core_verify`. fn key_is_identity(&self) -> bool { - self.pubkey.to_bytes()[0] & INFINITY_FLAG != 0 + g1_point(&self.pubkey).is_infinity() } } -/// A random weight in `1..2^64`, as the little-endian scalar `blst` expects. -fn random_weight(rng: &mut impl RngExt) -> blst_scalar { - let weight = loop { +/// A random weight in `1..2^64`. +fn random_weight(rng: &mut impl RngExt) -> u64 { + loop { let candidate: u64 = rng.random(); if candidate != 0 { - break candidate; + return candidate; } - }; - let mut scalar = blst_scalar::default(); - scalar.b[..8].copy_from_slice(&weight.to_le_bytes()); - scalar + } } -/// Whether every set verifies, in one random-linear-combination check: -/// `verify_multiple_aggregate_signatures` with 64-bit scalars from the -/// OS-seeded RNG and one final exponentiation. An empty slice verifies. +/// Whether every set verifies, in random-linear-combination checks with +/// 64-bit weights from the OS-seeded RNG, one per chunk of [`SETS_PER_CHUNK`] +/// sets, each with one final exponentiation. An empty slice verifies. /// -/// See the module documentation for why the weights are random. A single set -/// skips them and is verified directly, since one equation needs no -/// combining. A set whose key is the identity fails the batch, as it fails -/// [`SignatureSet::verify`]. blst rejects that input itself, so the explicit -/// check only states the rule where a reader of the batch looks for it. +/// See the module documentation for why the weights are random, where they are +/// applied, and why the batch is chunked. A single set skips them and is verified directly, since one +/// equation needs no combining. A set whose key is the identity fails the +/// batch, as it fails [`SignatureSet::verify`]: the pairing product would drop +/// that set's pair rather than refuse it. pub fn verify_signature_sets(sets: &[SignatureSet]) -> bool { match sets { [] => return true, @@ -201,23 +194,38 @@ pub fn verify_signature_sets(sets: &[SignatureSet]) -> bool { if sets.iter().any(SignatureSet::key_is_identity) { return false; } + sets.par_chunks(SETS_PER_CHUNK).all(chunk_holds) +} + +/// How many sets one chunk of a batch holds: with the pair for the chunk's +/// signature sum, exactly one pass of the pairing product's Miller-loop lanes. +const SETS_PER_CHUNK: usize = LANES - 1; + +/// One chunk's random linear combination, with its own weights; see the module +/// documentation. Every key is already known not to be the identity. +fn chunk_holds(sets: &[SignatureSet]) -> bool { let mut rng = rand::rng(); - let weights: Vec = sets.iter().map(|_| random_weight(&mut rng)).collect(); - let messages: Vec<&[u8]> = sets.iter().map(|set| set.message.as_slice()).collect(); - let pubkeys: Vec<&PublicKey> = sets.iter().map(|set| &set.pubkey).collect(); - let signatures: Vec<&Signature> = sets.iter().map(|set| &set.signature).collect(); - // Keys and signatures were validated when the sets were built. - let result = Signature::verify_multiple_aggregate_signatures( - &messages, - DST, - &pubkeys, - false, - &signatures, - false, - &weights, - RAND_BITS, - ); - result == BLST_ERROR::BLST_SUCCESS + let weights: Vec = sets.iter().map(|_| random_weight(&mut rng)).collect(); + // Hashing a message and weighting a key are each set's own work and the + // costly part of a chunk, so they run in parallel. + let pairs: Vec<(G1Point, G2Point)> = sets + .par_iter() + .zip(&weights) + .map(|(set, weight)| { + let key = G1Point::weighted_sum(&[g1_point(&set.pubkey)], &weight.to_le_bytes()); + (key, G2Point::hash_to_curve(set.message.as_slice(), DST)) + }) + .collect(); + let scalars: Vec = weights.iter().flat_map(|w| w.to_le_bytes()).collect(); + let signatures: Vec = sets.iter().map(|set| set.signature).collect(); + let signature_sum = G2Point::weighted_sum(&signatures, &scalars); + + let mut product = PairingProduct::with_capacity(sets.len() + 1); + product.push(&G1Point::negative_generator(), &signature_sum); + for (key, hash) in &pairs { + product.push(key, hash); + } + product.is_one() } #[cfg(test)] @@ -426,13 +434,53 @@ mod tests { assert!(verify_signature_sets(&sets)); } + /// The batch answers what checking every set alone through `blst` answers, + /// at batch sizes on both sides of the pairing kernel's eight lanes (the + /// batch adds one pair for the signature sum), with no bad set and with a + /// bad set in each position class. + #[test] + fn a_batch_agrees_with_blst_checking_each_set_alone() { + use blst::BLST_ERROR::BLST_SUCCESS; + for n in [2u8, 7, 8, 9, 15, 16, 17] { + for bad in [None, Some(0), Some(n / 2), Some(n - 1)] { + let mut inputs: Vec<(BlsPubkey, Root, BlsSignature)> = (0..n) + .map(|i| { + let key = secret_key(100 + i); + let msg = message(i); + (pubkey_of(&key), msg, sign(&key, msg)) + }) + .collect(); + if let Some(bad) = bad { + let key = secret_key(100 + bad); + inputs[usize::from(bad)].2 = sign(&key, message(255)); + } + let theirs = inputs.iter().all(|(pubkey, msg, signature)| { + let key = PublicKey::key_validate(pubkey.as_ref()).unwrap(); + let sig = + blst::min_pk::Signature::sig_validate(signature.as_ref(), false).unwrap(); + sig.verify(false, msg.as_slice(), DST, &[], &key, false) == BLST_SUCCESS + }); + assert_eq!(theirs, bad.is_none()); + let sets: Vec = inputs + .iter() + .map(|(pubkey, msg, signature)| { + SignatureSet::single(pubkey, *msg, signature).unwrap() + }) + .collect(); + assert_eq!( + verify_signature_sets(&sets), + theirs, + "{n} sets, bad {bad:?}" + ); + } + } + } + #[test] fn random_weights_are_never_zero() { let mut rng = rand::rng(); for _ in 0..10_000 { - let weight = random_weight(&mut rng); - assert!(weight.b[..8].iter().any(|byte| *byte != 0)); - assert!(weight.b[8..].iter().all(|byte| *byte == 0)); + assert_ne!(random_weight(&mut rng), 0); } } } diff --git a/crates/common/crypto/src/bls/committee_sums.rs b/crates/common/crypto/src/bls/committee_sums.rs index 7416c7347..c15eef439 100644 --- a/crates/common/crypto/src/bls/committee_sums.rs +++ b/crates/common/crypto/src/bls/committee_sums.rs @@ -31,8 +31,8 @@ use super::{PubkeySource, SignatureSet, sum_validated_projective, validated_pubk /// Not a [`BlsPubkey`] on purpose: compressing a point costs a field inversion, /// and the only consumer is a pairing check in this process, so the point stays /// as it is. The sum may be the identity (every key subtracted away), which is -/// representable here and refused by [`Self::verify`], exactly as `blst` -/// refuses an infinite key at pairing time. +/// representable here and refused by [`Self::verify`] before any pairing, as +/// an identity key summed from scratch is. #[derive(Clone, Copy)] pub struct AggregatePubkey(AggregatePublicKey); @@ -76,7 +76,7 @@ impl AggregatePubkey { /// [`super::fast_aggregate_verify`] validates it: nothing upstream /// guarantees a subgroup-correct point. The key needs no check, since a sum /// of validated points is in the subgroup; the identity is the one value - /// that is not a valid key, and `blst` rejects it at pairing time. + /// that is not a valid key, and the pairing check refuses it. pub fn verify(&self, message: Root, signature: &BlsSignature) -> bool { SignatureSet::from_aggregate(*self, message, signature).is_some_and(|set| set.verify()) }