From 58d16bd0748c57332e9467e67ba849c44a56212e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:23:36 -0300 Subject: [PATCH 1/5] feat!: use Keccak-256 instead of SHA-256 for hash_tree_root Swap the `sha2` backend for `sha3::Keccak256` (legacy Keccak padding, as used by Ethereum's execution layer), including the build-time ZERO_HASHES table so zero-subtree elision stays consistent with `hash_nodes`. Breaking renames: - trait `Sha256Hasher` -> `MerkleHasher` - struct `Sha2Hasher` -> `KeccakHasher` - feature `sha2-backend` -> `keccak-backend` Roots are no longer consensus-compatible: the two unit tests pinned to SHA-256 reference roots are ignored, and spec-tests / differential fuzz targets against Lighthouse are expected to fail. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011vghNkypyMGfhziz7iorub --- benches/benches/differential.rs | 16 +- benches/benches/hash_tree_root.rs | 58 ++-- benches/benches/iai_merkle.rs | 8 +- benches/benches/merkle.rs | 10 +- benches/src/fixtures.rs | 4 +- crates/ssz-derive/src/lib.rs | 8 +- crates/ssz-derive/tests/derive_tests.rs | 52 +-- crates/ssz-merkle/Cargo.toml | 8 +- crates/ssz-merkle/build.rs | 4 +- crates/ssz-merkle/src/ethereum_types.rs | 48 +-- crates/ssz-merkle/src/lib.rs | 307 ++++++++++-------- crates/ssz-types/src/hash_tree_root.rs | 66 ++-- crates/ssz-types/src/progressive_bitlist.rs | 4 +- crates/ssz-types/src/progressive_list.rs | 4 +- docs/technical-design.md | 8 +- fuzz/fuzz_targets/diff_hash_tree_root.rs | 2 +- fuzz/fuzz_targets/diff_htr_container.rs | 2 +- fuzz/fuzz_targets/hash_tree_root_fuzz.rs | 40 +-- fuzz/fuzz_targets/merkle_primitives.rs | 6 +- fuzz/fuzz_targets/union_htr.rs | 22 +- spec-tests/src/types/compatible_unions.rs | 8 +- .../src/types/progressive_containers.rs | 6 +- spec-tests/tests/ssz_generic.rs | 30 +- spec-tests/tests/ssz_static.rs | 2 +- spec-tests/tests/ssz_static_minimal.rs | 2 +- 25 files changed, 372 insertions(+), 353 deletions(-) diff --git a/benches/benches/differential.rs b/benches/benches/differential.rs index f22580a..5caf8a4 100644 --- a/benches/benches/differential.rs +++ b/benches/benches/differential.rs @@ -987,7 +987,7 @@ fn diff_htr(c: &mut Criterion) { // bool group.bench_function("libssz/bool", |b| { - b.iter(|| black_box(true).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(true).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("lighthouse/bool", |b| { b.iter(|| tree_hash::TreeHash::tree_hash_root(black_box(&true)).0) @@ -1002,7 +1002,7 @@ fn diff_htr(c: &mut Criterion) { // u64 let val_u64 = 0x1234_5678_9ABC_DEF0u64; group.bench_function("libssz/u64", |b| { - b.iter(|| black_box(val_u64).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(val_u64).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("lighthouse/u64", |b| { b.iter(|| tree_hash::TreeHash::tree_hash_root(black_box(&val_u64)).0) @@ -1017,7 +1017,7 @@ fn diff_htr(c: &mut Criterion) { // [u8; 32] let val_bytes32 = [0xABu8; 32]; group.bench_function("libssz/bytes32", |b| { - b.iter(|| black_box(&val_bytes32).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&val_bytes32).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("lighthouse/bytes32", |b| { b.iter(|| tree_hash::TreeHash::tree_hash_root(black_box(&val_bytes32)).0) @@ -1149,22 +1149,22 @@ fn diff_htr_consensus_containers(c: &mut Criterion) { let fork = make_fork(); group.bench_function("libssz/fork", |b| { - b.iter(|| black_box(&fork).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&fork).hash_tree_root(&libssz_merkle::KeccakHasher)) }); let checkpoint = make_checkpoint(42); group.bench_function("libssz/checkpoint", |b| { - b.iter(|| black_box(&checkpoint).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&checkpoint).hash_tree_root(&libssz_merkle::KeccakHasher)) }); let eth1_data = make_eth1_data(42); group.bench_function("libssz/eth1_data", |b| { - b.iter(|| black_box(ð1_data).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(ð1_data).hash_tree_root(&libssz_merkle::KeccakHasher)) }); let attestation_data = make_attestation_data(42); group.bench_function("libssz/attestation_data", |b| { - b.iter(|| black_box(&attestation_data).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&attestation_data).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.finish(); @@ -1283,7 +1283,7 @@ fn diff_htr_beacon_state(c: &mut Criterion) { BenchmarkId::new("libssz", n_validators), &state, |b, state| { - b.iter(|| black_box(state).hash_tree_root(&libssz_merkle::Sha2Hasher)); + b.iter(|| black_box(state).hash_tree_root(&libssz_merkle::KeccakHasher)); }, ); group.bench_with_input( diff --git a/benches/benches/hash_tree_root.rs b/benches/benches/hash_tree_root.rs index 684221c..c37106e 100644 --- a/benches/benches/hash_tree_root.rs +++ b/benches/benches/hash_tree_root.rs @@ -10,25 +10,25 @@ use ssz_bench::fixtures::{ fn htr_primitives(c: &mut Criterion) { let mut group = c.benchmark_group("hash_tree_root/primitives"); group.bench_function("bool", |b| { - b.iter(|| black_box(true).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(true).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("u8", |b| { - b.iter(|| black_box(0xABu8).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(0xABu8).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("u16", |b| { - b.iter(|| black_box(0x1234u16).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(0x1234u16).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("u32", |b| { - b.iter(|| black_box(0x1234_5678u32).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(0x1234_5678u32).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("u64", |b| { - b.iter(|| black_box(0x1234_5678_9abc_def0u64).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(0x1234_5678_9abc_def0u64).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("u128", |b| { - b.iter(|| black_box(u128::MAX).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(u128::MAX).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("bytes32", |b| { - b.iter(|| black_box([0xabu8; 32]).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box([0xabu8; 32]).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.finish(); } @@ -39,7 +39,7 @@ fn htr_vec_u64(c: &mut Criterion) { let data = make_vec_u64(size); group.throughput(Throughput::Bytes((size * 8) as u64)); group.bench_with_input(BenchmarkId::from_parameter(size), &data, |b, data| { - b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::Sha2Hasher)); + b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::KeccakHasher)); }); } group.finish(); @@ -52,7 +52,7 @@ fn htr_list_validator(c: &mut Criterion) { let list = make_validator_list(size); group.throughput(Throughput::Bytes((size * 121) as u64)); group.bench_with_input(BenchmarkId::from_parameter(size), &list, |b, list| { - b.iter(|| black_box(list).hash_tree_root(&libssz_merkle::Sha2Hasher)); + b.iter(|| black_box(list).hash_tree_root(&libssz_merkle::KeccakHasher)); }); } group.finish(); @@ -64,11 +64,11 @@ fn htr_vector_bytes32(c: &mut Criterion) { let v1000 = make_vector_bytes32::<1000>(); group.throughput(Throughput::Bytes((100 * 32) as u64)); group.bench_function("100", |b| { - b.iter(|| black_box(&v100).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&v100).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.throughput(Throughput::Bytes((1000 * 32) as u64)); group.bench_function("1000", |b| { - b.iter(|| black_box(&v1000).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&v1000).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.finish(); } @@ -81,33 +81,33 @@ fn htr_bitfields(c: &mut Criterion) { 64 => { let bl = make_bitlist::<64>(64); group.bench_with_input(BenchmarkId::new("bitlist", bits), &bl, |b, bl| { - b.iter(|| black_box(bl).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(bl).hash_tree_root(&libssz_merkle::KeccakHasher)) }); let bv = make_bitvector::<64>(); group.bench_with_input(BenchmarkId::new("bitvector", bits), &bv, |b, bv| { - b.iter(|| black_box(bv).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(bv).hash_tree_root(&libssz_merkle::KeccakHasher)) }); } 512 => { let bv = make_bitvector_512(); group.bench_with_input(BenchmarkId::new("bitvector", bits), &bv, |b, bv| { - b.iter(|| black_box(bv).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(bv).hash_tree_root(&libssz_merkle::KeccakHasher)) }); } 2048 => { let bl = make_bitlist_2048(); group.bench_with_input(BenchmarkId::new("bitlist", bits), &bl, |b, bl| { - b.iter(|| black_box(bl).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(bl).hash_tree_root(&libssz_merkle::KeccakHasher)) }); } 4096 => { let bl = make_bitlist::<4096>(4096); group.bench_with_input(BenchmarkId::new("bitlist", bits), &bl, |b, bl| { - b.iter(|| black_box(bl).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(bl).hash_tree_root(&libssz_merkle::KeccakHasher)) }); let bv = make_bitvector::<4096>(); group.bench_with_input(BenchmarkId::new("bitvector", bits), &bv, |b, bv| { - b.iter(|| black_box(bv).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(bv).hash_tree_root(&libssz_merkle::KeccakHasher)) }); } _ => unreachable!(), @@ -121,10 +121,10 @@ fn htr_containers(c: &mut Criterion) { let validator = make_validator(42); let header = make_header(42); group.bench_function("validator", |b| { - b.iter(|| black_box(&validator).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&validator).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("beacon_block_header", |b| { - b.iter(|| black_box(&header).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&header).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.finish(); } @@ -134,7 +134,7 @@ fn htr_unions(c: &mut Criterion) { for variant in 0..4 { let data = make_bench_union(variant); group.bench_with_input(BenchmarkId::from_parameter(variant), &data, |b, data| { - b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::KeccakHasher)) }); } group.finish(); @@ -146,7 +146,7 @@ fn htr_variable_container(c: &mut Criterion) { let data = make_variable_container(var_size); group.throughput(Throughput::Bytes((8 + var_size + 32 + var_size * 8) as u64)); group.bench_with_input(BenchmarkId::from_parameter(var_size), &data, |b, data| { - b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::KeccakHasher)) }); } group.finish(); @@ -159,7 +159,7 @@ fn htr_nested_container(c: &mut Criterion) { let data = make_nested_container(n); group.throughput(Throughput::Bytes((n * 121 + 120) as u64)); group.bench_with_input(BenchmarkId::from_parameter(n), &data, |b, data| { - b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::Sha2Hasher)); + b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::KeccakHasher)); }); } group.finish(); @@ -171,7 +171,7 @@ fn htr_list_u64(c: &mut Criterion) { let data = make_list_u64(size); group.throughput(Throughput::Bytes((size * 8) as u64)); group.bench_with_input(BenchmarkId::from_parameter(size), &data, |b, data| { - b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::Sha2Hasher)); + b.iter(|| black_box(data).hash_tree_root(&libssz_merkle::KeccakHasher)); }); } group.finish(); @@ -185,19 +185,19 @@ fn htr_consensus_containers(c: &mut Criterion) { let attestation_data = make_attestation_data(42); let pending_attestation = make_pending_attestation(42); group.bench_function("fork", |b| { - b.iter(|| black_box(&fork).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&fork).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("checkpoint", |b| { - b.iter(|| black_box(&checkpoint).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&checkpoint).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("eth1_data", |b| { - b.iter(|| black_box(ð1_data).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(ð1_data).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("attestation_data", |b| { - b.iter(|| black_box(&attestation_data).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&attestation_data).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.bench_function("pending_attestation", |b| { - b.iter(|| black_box(&pending_attestation).hash_tree_root(&libssz_merkle::Sha2Hasher)) + b.iter(|| black_box(&pending_attestation).hash_tree_root(&libssz_merkle::KeccakHasher)) }); group.finish(); } @@ -211,7 +211,7 @@ fn htr_beacon_state(c: &mut Criterion) { BenchmarkId::from_parameter(n_validators), &state, |b, state| { - b.iter(|| black_box(state).hash_tree_root(&libssz_merkle::Sha2Hasher)); + b.iter(|| black_box(state).hash_tree_root(&libssz_merkle::KeccakHasher)); }, ); } diff --git a/benches/benches/iai_merkle.rs b/benches/benches/iai_merkle.rs index 4f731c2..7426ef0 100644 --- a/benches/benches/iai_merkle.rs +++ b/benches/benches/iai_merkle.rs @@ -1,5 +1,5 @@ use iai_callgrind::{library_benchmark, library_benchmark_group, main}; -use libssz_merkle::{hash_nodes, merkleize, pack, Node, Sha2Hasher}; +use libssz_merkle::{hash_nodes, merkleize, pack, KeccakHasher, Node}; fn make_chunks(n: usize) -> Vec { (0..n) .map(|i| { @@ -14,19 +14,19 @@ fn make_chunks(n: usize) -> Vec { fn iai_hash_nodes() -> Node { let a: Node = [0xaa; 32]; let b: Node = [0xbb; 32]; - hash_nodes(&Sha2Hasher, &a, &b) + hash_nodes(&KeccakHasher, &a, &b) } #[library_benchmark] fn iai_merkleize_64() -> Node { let chunks = make_chunks(64); - merkleize(&Sha2Hasher, &chunks, None) + merkleize(&KeccakHasher, &chunks, None) } #[library_benchmark] fn iai_merkleize_1024() -> Node { let chunks = make_chunks(1024); - merkleize(&Sha2Hasher, &chunks, None) + merkleize(&KeccakHasher, &chunks, None) } #[library_benchmark] diff --git a/benches/benches/merkle.rs b/benches/benches/merkle.rs index 9c51a04..9261315 100644 --- a/benches/benches/merkle.rs +++ b/benches/benches/merkle.rs @@ -1,5 +1,5 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use libssz_merkle::{hash_nodes, merkleize, mix_in_length, pack, pack_bits, Node, Sha2Hasher}; +use libssz_merkle::{hash_nodes, merkleize, mix_in_length, pack, pack_bits, KeccakHasher, Node}; fn make_chunks(n: usize) -> Vec { (0..n) .map(|i| { @@ -15,7 +15,7 @@ fn bench_hash_nodes(c: &mut Criterion) { let a: Node = [0xaa; 32]; let b: Node = [0xbb; 32]; group.bench_function("single", |bench| { - bench.iter(|| hash_nodes(&Sha2Hasher, black_box(&a), black_box(&b))) + bench.iter(|| hash_nodes(&KeccakHasher, black_box(&a), black_box(&b))) }); group.finish(); } @@ -52,7 +52,7 @@ fn bench_merkleize(c: &mut Criterion) { for &n in &[1, 4, 16, 64, 256, 1024, 4096] { let chunks = make_chunks(n); group.bench_with_input(BenchmarkId::from_parameter(n), &chunks, |b, chunks| { - b.iter(|| merkleize(&Sha2Hasher, black_box(chunks), None)); + b.iter(|| merkleize(&KeccakHasher, black_box(chunks), None)); }); } group.finish(); @@ -73,7 +73,7 @@ fn bench_merkleize_with_limit(c: &mut Criterion) { let chunks = make_chunks(n); let label = format!("{n}_of_{limit}"); group.bench_with_input(BenchmarkId::new("chunks", &label), &chunks, |b, chunks| { - b.iter(|| merkleize(&Sha2Hasher, black_box(chunks), Some(limit))); + b.iter(|| merkleize(&KeccakHasher, black_box(chunks), Some(limit))); }); } group.finish(); @@ -83,7 +83,7 @@ fn bench_mix_in_length(c: &mut Criterion) { let mut group = c.benchmark_group("merkle/mix_in_length"); let root: Node = [0xcc; 32]; group.bench_function("single", |b| { - b.iter(|| mix_in_length(&Sha2Hasher, black_box(&root), black_box(1_000_000))) + b.iter(|| mix_in_length(&KeccakHasher, black_box(&root), black_box(1_000_000))) }); group.finish(); } diff --git a/benches/src/fixtures.rs b/benches/src/fixtures.rs index 6d25695..a295f38 100644 --- a/benches/src/fixtures.rs +++ b/benches/src/fixtures.rs @@ -1,6 +1,6 @@ use libssz::SszEncode; use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; -use libssz_merkle::{merkleize, mix_in_length, HashTreeRoot, Node, Sha256Hasher}; +use libssz_merkle::{merkleize, mix_in_length, HashTreeRoot, MerkleHasher, Node}; use libssz_types::{SszBitlist, SszBitvector, SszList, SszVector}; /// Ethereum consensus Validator (121 bytes, all fixed-size fields). @@ -336,7 +336,7 @@ pub struct NestedContainer { } impl HashTreeRoot for NestedContainer { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { let header_root = self.header.hash_tree_root(hasher); let validator_roots: Vec = self .validators diff --git a/crates/ssz-derive/src/lib.rs b/crates/ssz-derive/src/lib.rs index c4f1793..2985847 100644 --- a/crates/ssz-derive/src/lib.rs +++ b/crates/ssz-derive/src/lib.rs @@ -853,7 +853,7 @@ fn derive_htr_transparent( quote! { impl #impl_generics libssz_merkle::HashTreeRoot for #name #ty_generics #where_clause { - fn hash_tree_root(&self, hasher: &impl libssz_merkle::Sha256Hasher) -> libssz_merkle::Node { + fn hash_tree_root(&self, hasher: &impl libssz_merkle::MerkleHasher) -> libssz_merkle::Node { libssz_merkle::HashTreeRoot::hash_tree_root(&#field_access, hasher) } } @@ -873,7 +873,7 @@ fn derive_htr_struct( quote! { impl #impl_generics libssz_merkle::HashTreeRoot for #name #ty_generics #where_clause { - fn hash_tree_root(&self, hasher: &impl libssz_merkle::Sha256Hasher) -> libssz_merkle::Node { + fn hash_tree_root(&self, hasher: &impl libssz_merkle::MerkleHasher) -> libssz_merkle::Node { let field_roots: [libssz_merkle::Node; #num_fields] = [ #(#field_roots,)* ]; @@ -961,7 +961,7 @@ fn derive_htr_progressive_container( quote! { impl #impl_generics libssz_merkle::HashTreeRoot for #name #ty_generics #where_clause { - fn hash_tree_root(&self, hasher: &impl libssz_merkle::Sha256Hasher) -> libssz_merkle::Node { + fn hash_tree_root(&self, hasher: &impl libssz_merkle::MerkleHasher) -> libssz_merkle::Node { const ACTIVE_FIELDS: libssz_merkle::Node = [#(#active_node_bytes),*]; let chunks: [libssz_merkle::Node; #num_chunks] = [ #(#chunks,)* @@ -1011,7 +1011,7 @@ fn derive_htr_union_enum( quote! { impl #impl_generics libssz_merkle::HashTreeRoot for #name #ty_generics #where_clause { - fn hash_tree_root(&self, hasher: &impl libssz_merkle::Sha256Hasher) -> libssz_merkle::Node { + fn hash_tree_root(&self, hasher: &impl libssz_merkle::MerkleHasher) -> libssz_merkle::Node { match self { #(#variant_arms)* } diff --git a/crates/ssz-derive/tests/derive_tests.rs b/crates/ssz-derive/tests/derive_tests.rs index efd7229..1f76e89 100644 --- a/crates/ssz-derive/tests/derive_tests.rs +++ b/crates/ssz-derive/tests/derive_tests.rs @@ -1,6 +1,6 @@ use libssz::{DecodeError, SszDecode, SszEncode}; use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; -use libssz_merkle::{HashTreeRoot, Sha2Hasher}; +use libssz_merkle::{HashTreeRoot, KeccakHasher}; // ── All-fixed struct ── #[derive(Debug, PartialEq, SszEncode, SszDecode)] @@ -138,12 +138,12 @@ struct SimpleContainer { #[test] fn hash_tree_root_struct() { let s = SimpleContainer { a: 1, b: 2 }; - let root = s.hash_tree_root(&Sha2Hasher); + let root = s.hash_tree_root(&KeccakHasher); // Manual: merkleize([hash_tree_root(a), hash_tree_root(b)]) - let root_a = 1u64.hash_tree_root(&Sha2Hasher); - let root_b = 2u64.hash_tree_root(&Sha2Hasher); - let expected = libssz_merkle::merkleize(&Sha2Hasher, &[root_a, root_b], None); + let root_a = 1u64.hash_tree_root(&KeccakHasher); + let root_b = 2u64.hash_tree_root(&KeccakHasher); + let expected = libssz_merkle::merkleize(&KeccakHasher, &[root_a, root_b], None); assert_eq!(root, expected); } @@ -203,10 +203,10 @@ fn union_decode_invalid_selector() { #[test] fn union_hash_tree_root() { let val = TestUnion::A(42); - let root = val.hash_tree_root(&Sha2Hasher); + let root = val.hash_tree_root(&KeccakHasher); - let inner_root = 42u32.hash_tree_root(&Sha2Hasher); - let expected = libssz_merkle::mix_in_selector(&Sha2Hasher, &inner_root, 0); + let inner_root = 42u32.hash_tree_root(&KeccakHasher); + let expected = libssz_merkle::mix_in_selector(&KeccakHasher, &inner_root, 0); assert_eq!(root, expected); } @@ -235,8 +235,8 @@ fn transparent_decode_round_trip() { fn transparent_hash_tree_root() { let val = Wrapper(42); assert_eq!( - val.hash_tree_root(&Sha2Hasher), - 42u64.hash_tree_root(&Sha2Hasher) + val.hash_tree_root(&KeccakHasher), + 42u64.hash_tree_root(&KeccakHasher) ); } @@ -269,16 +269,16 @@ struct ProgressiveThreeFields { #[test] fn hash_tree_root_progressive_container() { let s = ProgressiveThreeFields { a: 1, b: 2, c: 3 }; - let root = s.hash_tree_root(&Sha2Hasher); + let root = s.hash_tree_root(&KeccakHasher); // Manual: mix_in_active_fields(merkleize_progressive(field_roots), [1, 1, 1]) let field_roots = [ - 1u64.hash_tree_root(&Sha2Hasher), - 2u64.hash_tree_root(&Sha2Hasher), - 3u64.hash_tree_root(&Sha2Hasher), + 1u64.hash_tree_root(&KeccakHasher), + 2u64.hash_tree_root(&KeccakHasher), + 3u64.hash_tree_root(&KeccakHasher), ]; - let progressive = libssz_merkle::merkleize_progressive(&Sha2Hasher, &field_roots); - let expected = libssz_merkle::mix_in_active_fields(&Sha2Hasher, &progressive, &[true; 3]); + let progressive = libssz_merkle::merkleize_progressive(&KeccakHasher, &field_roots); + let expected = libssz_merkle::mix_in_active_fields(&KeccakHasher, &progressive, &[true; 3]); assert_eq!(root, expected); } @@ -298,8 +298,8 @@ fn progressive_container_root_differs_from_plain_container() { assert_eq!(progressive.to_ssz(), plain.to_ssz()); assert_ne!( - progressive.hash_tree_root(&Sha2Hasher), - plain.hash_tree_root(&Sha2Hasher) + progressive.hash_tree_root(&KeccakHasher), + plain.hash_tree_root(&KeccakHasher) ); } @@ -321,19 +321,19 @@ fn hash_tree_root_progressive_container_with_inactive_fields() { let zero = [0u8; 32]; let chunks = [ - 1u64.hash_tree_root(&Sha2Hasher), + 1u64.hash_tree_root(&KeccakHasher), zero, - 2u64.hash_tree_root(&Sha2Hasher), + 2u64.hash_tree_root(&KeccakHasher), zero, - 3u64.hash_tree_root(&Sha2Hasher), + 3u64.hash_tree_root(&KeccakHasher), ]; - let progressive = libssz_merkle::merkleize_progressive(&Sha2Hasher, &chunks); + let progressive = libssz_merkle::merkleize_progressive(&KeccakHasher, &chunks); let expected = libssz_merkle::mix_in_active_fields( - &Sha2Hasher, + &KeccakHasher, &progressive, &[true, false, true, false, true], ); - assert_eq!(s.hash_tree_root(&Sha2Hasher), expected); + assert_eq!(s.hash_tree_root(&KeccakHasher), expected); } /// `active_fields` changes only merkleization: the serialization is the same as @@ -345,7 +345,7 @@ fn inactive_fields_change_root_but_not_encoding() { assert_eq!(inactive.to_ssz(), all_active.to_ssz()); assert_ne!( - inactive.hash_tree_root(&Sha2Hasher), - all_active.hash_tree_root(&Sha2Hasher) + inactive.hash_tree_root(&KeccakHasher), + all_active.hash_tree_root(&KeccakHasher) ); } diff --git a/crates/ssz-merkle/Cargo.toml b/crates/ssz-merkle/Cargo.toml index cae1346..b5a7381 100644 --- a/crates/ssz-merkle/Cargo.toml +++ b/crates/ssz-merkle/Cargo.toml @@ -11,15 +11,15 @@ categories = ["encoding", "cryptography::cryptocurrencies"] [features] default = ["std"] -std = ["alloc", "libssz/std", "sha2-backend"] +std = ["alloc", "libssz/std", "keccak-backend"] alloc = ["libssz/alloc"] -sha2-backend = ["dep:sha2"] +keccak-backend = ["dep:sha3"] ethereum_types = ["dep:ethereum-types", "libssz/ethereum_types"] [dependencies] libssz = { version = "0.3.0", path = "../ssz", default-features = false } -sha2 = { version = "0.10", default-features = false, optional = true } +sha3 = { version = "0.10", default-features = false, optional = true } ethereum-types = { version = "0.15", default-features = false, optional = true } [build-dependencies] -sha2 = "0.10" +sha3 = "0.10" diff --git a/crates/ssz-merkle/build.rs b/crates/ssz-merkle/build.rs index fe8cfba..bf7fed0 100644 --- a/crates/ssz-merkle/build.rs +++ b/crates/ssz-merkle/build.rs @@ -1,4 +1,4 @@ -use sha2::{Digest, Sha256}; +use sha3::{Digest, Keccak256}; use std::env; use std::fs; use std::path::Path; @@ -13,7 +13,7 @@ fn main() { // zero_hashes[0] is already all zeros for i in 1..=MAX_DEPTH { - let mut hasher = Sha256::new(); + let mut hasher = Keccak256::new(); hasher.update(zero_hashes[i - 1]); hasher.update(zero_hashes[i - 1]); let result = hasher.finalize(); diff --git a/crates/ssz-merkle/src/ethereum_types.rs b/crates/ssz-merkle/src/ethereum_types.rs index 87d1aeb..e9e550b 100644 --- a/crates/ssz-merkle/src/ethereum_types.rs +++ b/crates/ssz-merkle/src/ethereum_types.rs @@ -2,12 +2,12 @@ use ethereum_types as et; -use crate::{HashTreeRoot, Node, Sha256Hasher}; +use crate::{HashTreeRoot, MerkleHasher, Node}; // ── H-types: delegate to inner [u8; N] ── impl HashTreeRoot for et::H32 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.as_fixed_bytes().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -16,7 +16,7 @@ impl HashTreeRoot for et::H32 { } impl HashTreeRoot for et::H64 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.as_fixed_bytes().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -25,7 +25,7 @@ impl HashTreeRoot for et::H64 { } impl HashTreeRoot for et::H128 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.as_fixed_bytes().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -34,7 +34,7 @@ impl HashTreeRoot for et::H128 { } impl HashTreeRoot for et::H160 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.as_fixed_bytes().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -43,7 +43,7 @@ impl HashTreeRoot for et::H160 { } impl HashTreeRoot for et::H256 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.as_fixed_bytes().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -52,7 +52,7 @@ impl HashTreeRoot for et::H256 { } impl HashTreeRoot for et::H264 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.as_fixed_bytes().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -61,7 +61,7 @@ impl HashTreeRoot for et::H264 { } impl HashTreeRoot for et::H512 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.as_fixed_bytes().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -70,7 +70,7 @@ impl HashTreeRoot for et::H512 { } impl HashTreeRoot for et::H520 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.as_fixed_bytes().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -81,7 +81,7 @@ impl HashTreeRoot for et::H520 { // ── U-types: serialize to LE bytes, then hash ── impl HashTreeRoot for et::U64 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.to_little_endian().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -90,7 +90,7 @@ impl HashTreeRoot for et::U64 { } impl HashTreeRoot for et::U128 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.to_little_endian().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -99,7 +99,7 @@ impl HashTreeRoot for et::U128 { } impl HashTreeRoot for et::U256 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.to_little_endian().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -108,7 +108,7 @@ impl HashTreeRoot for et::U256 { } impl HashTreeRoot for et::U512 { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { self.to_little_endian().hash_tree_root(hasher) } fn is_basic_type() -> bool { @@ -119,7 +119,7 @@ impl HashTreeRoot for et::U512 { #[cfg(test)] mod tests { use super::*; - use crate::{merkleize, pack, Sha2Hasher}; + use crate::{merkleize, pack, KeccakHasher}; // H-type basic type checks #[test] @@ -158,8 +158,8 @@ mod tests { let bytes = [0xab_u8; 32]; let hval = et::H256::from_slice(&bytes); assert_eq!( - hval.hash_tree_root(&Sha2Hasher), - bytes.hash_tree_root(&Sha2Hasher) + hval.hash_tree_root(&KeccakHasher), + bytes.hash_tree_root(&KeccakHasher) ); } @@ -168,8 +168,8 @@ mod tests { let bytes = [0xcd_u8; 20]; let hval = et::H160::from_slice(&bytes); assert_eq!( - hval.hash_tree_root(&Sha2Hasher), - bytes.hash_tree_root(&Sha2Hasher) + hval.hash_tree_root(&KeccakHasher), + bytes.hash_tree_root(&KeccakHasher) ); } @@ -178,8 +178,8 @@ mod tests { let bytes = [0xef_u8; 64]; let hval = et::H512::from_slice(&bytes); assert_eq!( - hval.hash_tree_root(&Sha2Hasher), - bytes.hash_tree_root(&Sha2Hasher) + hval.hash_tree_root(&KeccakHasher), + bytes.hash_tree_root(&KeccakHasher) ); } @@ -203,7 +203,7 @@ mod tests { #[test] fn u256_htr_is_le_bytes() { let val = et::U256::from(42); - let root = val.hash_tree_root(&Sha2Hasher); + let root = val.hash_tree_root(&KeccakHasher); let expected = val.to_little_endian(); assert_eq!(root, expected); } @@ -211,7 +211,7 @@ mod tests { #[test] fn u64_htr_is_padded_le() { let val = et::U64::from(1); - let root = val.hash_tree_root(&Sha2Hasher); + let root = val.hash_tree_root(&KeccakHasher); let mut expected = [0u8; 32]; expected[0] = 1; assert_eq!(root, expected); @@ -221,7 +221,7 @@ mod tests { fn u512_htr_is_merkleized() { let val = et::U512::from(42); let bytes = val.to_little_endian(); - let expected = merkleize(&Sha2Hasher, &pack(&bytes), None); - assert_eq!(val.hash_tree_root(&Sha2Hasher), expected); + let expected = merkleize(&KeccakHasher, &pack(&bytes), None); + assert_eq!(val.hash_tree_root(&KeccakHasher), expected); } } diff --git a/crates/ssz-merkle/src/lib.rs b/crates/ssz-merkle/src/lib.rs index f175298..b97facb 100644 --- a/crates/ssz-merkle/src/lib.rs +++ b/crates/ssz-merkle/src/lib.rs @@ -16,29 +16,35 @@ pub type Node = [u8; 32]; include!(concat!(env!("OUT_DIR"), "/zero_hashes.rs")); -/// Trait for pluggable SHA-256 implementations. +/// Trait for pluggable Keccak-256 implementations. /// -/// Instance method (`&self`) enables passing `&impl Sha256Hasher` through the +/// Instance method (`&self`) enables passing `&impl MerkleHasher` through the /// call chain. The hasher struct is typically zero-sized so `&self` compiles away. -pub trait Sha256Hasher { +/// +/// Implementations must compute Keccak-256: [`ZERO_HASHES`] is precomputed with +/// it, and [`merkleize`] substitutes those entries for all-zero subtrees. +pub trait MerkleHasher { fn hash(&self, data: &[u8]) -> [u8; 32]; } -/// Default SHA-256 hasher using the `sha2` crate. -#[cfg(feature = "sha2-backend")] -pub struct Sha2Hasher; +/// Default Keccak-256 hasher using the `sha3` crate. +/// +/// This is the original Keccak-256 used by Ethereum's execution layer (`0x01` +/// padding), not NIST SHA3-256. +#[cfg(feature = "keccak-backend")] +pub struct KeccakHasher; -#[cfg(feature = "sha2-backend")] -impl Sha256Hasher for Sha2Hasher { +#[cfg(feature = "keccak-backend")] +impl MerkleHasher for KeccakHasher { fn hash(&self, data: &[u8]) -> [u8; 32] { - use sha2::{Digest, Sha256}; - Sha256::digest(data).into() + use sha3::{Digest, Keccak256}; + Keccak256::digest(data).into() } } -/// Hash two nodes together: SHA256(a || b). +/// Hash two nodes together: Keccak256(a || b). #[inline] -pub fn hash_nodes(hasher: &impl Sha256Hasher, a: &Node, b: &Node) -> Node { +pub fn hash_nodes(hasher: &impl MerkleHasher, a: &Node, b: &Node) -> Node { let mut buf = [0u8; 64]; buf[..32].copy_from_slice(a); buf[32..].copy_from_slice(b); @@ -81,7 +87,7 @@ pub fn pack_bits(bits: &[u8], num_bits: usize) -> Vec { /// Cost is `O(chunks.len())` plus one hash per virtual level above the real subtree. /// Two buffers are allocated and reused (≈ `chunks.len()` and `chunks.len().div_ceil(2)` nodes). #[cfg(feature = "alloc")] -pub fn merkleize(hasher: &impl Sha256Hasher, chunks: &[Node], limit: Option) -> Node { +pub fn merkleize(hasher: &impl MerkleHasher, chunks: &[Node], limit: Option) -> Node { let count = match limit { Some(l) => { assert!( @@ -149,13 +155,13 @@ pub fn merkleize(hasher: &impl Sha256Hasher, chunks: &[Node], limit: Option Node { +pub fn merkleize_progressive(hasher: &impl MerkleHasher, chunks: &[Node]) -> Node { merkleize_progressive_inner(hasher, chunks, 1) } #[cfg(feature = "alloc")] fn merkleize_progressive_inner( - hasher: &impl Sha256Hasher, + hasher: &impl MerkleHasher, chunks: &[Node], num_leaves: usize, ) -> Node { @@ -175,7 +181,7 @@ fn merkleize_progressive_inner( /// Mix in an active_fields bitvector: hash_nodes(root, pack_bits(active_fields)). #[cfg(feature = "alloc")] pub fn mix_in_active_fields( - hasher: &impl Sha256Hasher, + hasher: &impl MerkleHasher, root: &Node, active_fields: &[bool], ) -> Node { @@ -193,7 +199,7 @@ pub fn mix_in_active_fields( /// Mix in a length value: hash_nodes(root, length_as_le_bytes_node). #[inline] -pub fn mix_in_length(hasher: &impl Sha256Hasher, root: &Node, length: usize) -> Node { +pub fn mix_in_length(hasher: &impl MerkleHasher, root: &Node, length: usize) -> Node { let mut length_node = [0u8; 32]; length_node[..8].copy_from_slice(&(length as u64).to_le_bytes()); hash_nodes(hasher, root, &length_node) @@ -201,7 +207,7 @@ pub fn mix_in_length(hasher: &impl Sha256Hasher, root: &Node, length: usize) -> /// Mix in a selector value: hash_nodes(root, selector_as_le_bytes_node). #[inline] -pub fn mix_in_selector(hasher: &impl Sha256Hasher, root: &Node, selector: u8) -> Node { +pub fn mix_in_selector(hasher: &impl MerkleHasher, root: &Node, selector: u8) -> Node { let mut selector_node = [0u8; 32]; selector_node[0] = selector; hash_nodes(hasher, root, &selector_node) @@ -209,7 +215,7 @@ pub fn mix_in_selector(hasher: &impl Sha256Hasher, root: &Node, selector: u8) -> /// Trait for types that can compute their SSZ hash tree root. pub trait HashTreeRoot { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node; + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node; /// Returns `true` for SSZ basic types (bool, uN, byte arrays). /// Composite types (containers, vectors, lists) return `false`. @@ -226,7 +232,7 @@ pub trait HashTreeRoot { impl HashTreeRoot for bool { #[inline(always)] - fn hash_tree_root(&self, _hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, _hasher: &impl MerkleHasher) -> Node { let mut node = [0u8; 32]; node[0] = *self as u8; node @@ -241,7 +247,7 @@ impl HashTreeRoot for bool { impl HashTreeRoot for u8 { #[inline(always)] - fn hash_tree_root(&self, _hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, _hasher: &impl MerkleHasher) -> Node { let mut node = [0u8; 32]; let bytes = self.to_le_bytes(); node[..bytes.len()].copy_from_slice(&bytes); @@ -255,7 +261,7 @@ impl HashTreeRoot for u8 { impl HashTreeRoot for u16 { #[inline(always)] - fn hash_tree_root(&self, _hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, _hasher: &impl MerkleHasher) -> Node { let mut node = [0u8; 32]; let bytes = self.to_le_bytes(); node[..bytes.len()].copy_from_slice(&bytes); @@ -269,7 +275,7 @@ impl HashTreeRoot for u16 { impl HashTreeRoot for u32 { #[inline(always)] - fn hash_tree_root(&self, _hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, _hasher: &impl MerkleHasher) -> Node { let mut node = [0u8; 32]; let bytes = self.to_le_bytes(); node[..bytes.len()].copy_from_slice(&bytes); @@ -283,7 +289,7 @@ impl HashTreeRoot for u32 { impl HashTreeRoot for u64 { #[inline(always)] - fn hash_tree_root(&self, _hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, _hasher: &impl MerkleHasher) -> Node { let mut node = [0u8; 32]; let bytes = self.to_le_bytes(); node[..bytes.len()].copy_from_slice(&bytes); @@ -297,7 +303,7 @@ impl HashTreeRoot for u64 { impl HashTreeRoot for u128 { #[inline(always)] - fn hash_tree_root(&self, _hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, _hasher: &impl MerkleHasher) -> Node { let mut node = [0u8; 32]; let bytes = self.to_le_bytes(); node[..bytes.len()].copy_from_slice(&bytes); @@ -313,7 +319,7 @@ impl HashTreeRoot for u128 { #[cfg(feature = "alloc")] impl HashTreeRoot for [u8; N] { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { if N <= 32 { let mut node = [0u8; 32]; node[..N].copy_from_slice(self); @@ -332,7 +338,7 @@ impl HashTreeRoot for [u8; N] { #[cfg(feature = "alloc")] impl HashTreeRoot for Vec { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { let chunks: Vec = if T::is_basic_type() { let mut serialized = Vec::new(); for item in self { @@ -359,31 +365,42 @@ mod tests { use alloc::vec; use alloc::vec::Vec; - #[cfg(feature = "sha2-backend")] - use crate::Sha2Hasher; + #[cfg(feature = "keccak-backend")] + use crate::KeccakHasher; // ── hash_nodes tests ── - #[cfg(feature = "sha2-backend")] + /// Pins the legacy Keccak-256 padding (not NIST SHA3-256) via the + /// well-known empty-input digest. + #[cfg(feature = "keccak-backend")] + #[test] + fn keccak_hasher_is_legacy_keccak256() { + assert_eq!( + KeccakHasher.hash(&[]), + node_from_hex("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + ); + } + + #[cfg(feature = "keccak-backend")] #[test] fn hash_nodes_of_two_zero_nodes() { let zero = [0u8; 32]; - let result = hash_nodes(&Sha2Hasher, &zero, &zero); - use sha2::{Digest, Sha256}; - let mut sha = Sha256::new(); - sha.update([0u8; 64]); - let expected: [u8; 32] = sha.finalize().into(); + let result = hash_nodes(&KeccakHasher, &zero, &zero); + use sha3::{Digest, Keccak256}; + let mut keccak = Keccak256::new(); + keccak.update([0u8; 64]); + let expected: [u8; 32] = keccak.finalize().into(); assert_eq!(result, expected); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_nodes_is_not_commutative() { let a = [1u8; 32]; let b = [2u8; 32]; assert_ne!( - hash_nodes(&Sha2Hasher, &a, &b), - hash_nodes(&Sha2Hasher, &b, &a) + hash_nodes(&KeccakHasher, &a, &b), + hash_nodes(&KeccakHasher, &b, &a) ); } @@ -394,17 +411,17 @@ mod tests { assert_eq!(ZERO_HASHES[0], [0u8; 32]); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn zero_hash_level_1_is_hash_of_two_zeros() { - let expected = hash_nodes(&Sha2Hasher, &[0u8; 32], &[0u8; 32]); + let expected = hash_nodes(&KeccakHasher, &[0u8; 32], &[0u8; 32]); assert_eq!(ZERO_HASHES[1], expected); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn zero_hash_level_2_is_hash_of_two_level_1() { - let expected = hash_nodes(&Sha2Hasher, &ZERO_HASHES[1], &ZERO_HASHES[1]); + let expected = hash_nodes(&KeccakHasher, &ZERO_HASHES[1], &ZERO_HASHES[1]); assert_eq!(ZERO_HASHES[2], expected); } @@ -494,73 +511,73 @@ mod tests { // ── merkleize tests ── - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_empty_returns_zero_hash() { - let result = merkleize(&Sha2Hasher, &[], None); + let result = merkleize(&KeccakHasher, &[], None); assert_eq!(result, ZERO_HASHES[0]); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_one_chunk_is_identity() { let chunk = [0x42u8; 32]; - let result = merkleize(&Sha2Hasher, &[chunk], None); + let result = merkleize(&KeccakHasher, &[chunk], None); assert_eq!(result, chunk); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_two_chunks() { let a = [1u8; 32]; let b = [2u8; 32]; - let result = merkleize(&Sha2Hasher, &[a, b], None); - assert_eq!(result, hash_nodes(&Sha2Hasher, &a, &b)); + let result = merkleize(&KeccakHasher, &[a, b], None); + assert_eq!(result, hash_nodes(&KeccakHasher, &a, &b)); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_three_chunks_pads_to_four() { let a = [1u8; 32]; let b = [2u8; 32]; let c = [3u8; 32]; - let result = merkleize(&Sha2Hasher, &[a, b, c], None); + let result = merkleize(&KeccakHasher, &[a, b, c], None); // Tree: hash(hash(a,b), hash(c, zero)) - let left = hash_nodes(&Sha2Hasher, &a, &b); - let right = hash_nodes(&Sha2Hasher, &c, &ZERO_HASHES[0]); - assert_eq!(result, hash_nodes(&Sha2Hasher, &left, &right)); + let left = hash_nodes(&KeccakHasher, &a, &b); + let right = hash_nodes(&KeccakHasher, &c, &ZERO_HASHES[0]); + assert_eq!(result, hash_nodes(&KeccakHasher, &left, &right)); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_with_limit_pads_with_zero_hashes() { let a = [1u8; 32]; // With limit=4, single chunk should be padded to 4 leaves - let result = merkleize(&Sha2Hasher, &[a], Some(4)); + let result = merkleize(&KeccakHasher, &[a], Some(4)); // Tree: hash(hash(a, zero), hash(zero, zero)) - let left = hash_nodes(&Sha2Hasher, &a, &ZERO_HASHES[0]); + let left = hash_nodes(&KeccakHasher, &a, &ZERO_HASHES[0]); let right = ZERO_HASHES[1]; // hash(zero, zero) - assert_eq!(result, hash_nodes(&Sha2Hasher, &left, &right)); + assert_eq!(result, hash_nodes(&KeccakHasher, &left, &right)); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_with_limit_equal_to_count() { let a = [1u8; 32]; let b = [2u8; 32]; // limit=2 with 2 chunks should give same result as no limit assert_eq!( - merkleize(&Sha2Hasher, &[a, b], Some(2)), - merkleize(&Sha2Hasher, &[a, b], None) + merkleize(&KeccakHasher, &[a, b], Some(2)), + merkleize(&KeccakHasher, &[a, b], None) ); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] #[should_panic(expected = "chunk count")] fn merkleize_panics_if_count_exceeds_limit() { let chunks = [[0u8; 32]; 3]; - merkleize(&Sha2Hasher, &chunks, Some(2)); + merkleize(&KeccakHasher, &chunks, Some(2)); } // ── merkleize: zero-subtree elision ── @@ -568,7 +585,7 @@ mod tests { /// Naive merkleization: materializes every leaf up to `limit`, including the /// all-zero ones. Reference oracle for [`merkleize`], which must produce /// identical roots while skipping the zero subtrees. - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] fn merkleize_fully_materialized(chunks: &[Node], limit: Option) -> Node { let count = core::cmp::max(limit.unwrap_or(chunks.len()), chunks.len()); if count == 0 { @@ -579,7 +596,7 @@ mod tests { while layer.len() > 1 { layer = layer .chunks(2) - .map(|pair| hash_nodes(&Sha2Hasher, &pair[0], &pair[1])) + .map(|pair| hash_nodes(&KeccakHasher, &pair[0], &pair[1])) .collect(); } layer[0] @@ -587,12 +604,12 @@ mod tests { /// Counts `hash_nodes` invocations while delegating to the real hasher, so /// tree shape is unaffected. - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] struct CountingHasher { calls: core::cell::Cell, } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] impl CountingHasher { fn new() -> Self { Self { @@ -601,11 +618,11 @@ mod tests { } } - #[cfg(feature = "sha2-backend")] - impl Sha256Hasher for CountingHasher { + #[cfg(feature = "keccak-backend")] + impl MerkleHasher for CountingHasher { fn hash(&self, data: &[u8]) -> [u8; 32] { self.calls.set(self.calls.get() + 1); - Sha2Hasher.hash(data) + KeccakHasher.hash(data) } } @@ -629,13 +646,13 @@ mod tests { total } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_matches_fully_materialized_oracle() { for n in 0..=70 { let chunks: Vec = (0..n).map(|i| [i as u8 + 1; 32]).collect(); assert_eq!( - merkleize(&Sha2Hasher, &chunks, None), + merkleize(&KeccakHasher, &chunks, None), merkleize_fully_materialized(&chunks, None), "no limit, {n} chunks" ); @@ -644,7 +661,7 @@ mod tests { continue; } assert_eq!( - merkleize(&Sha2Hasher, &chunks, Some(limit)), + merkleize(&KeccakHasher, &chunks, Some(limit)), merkleize_fully_materialized(&chunks, Some(limit)), "limit {limit}, {n} chunks" ); @@ -652,7 +669,7 @@ mod tests { } } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_does_not_hash_zero_subtrees() { for n in [1usize, 2, 3, 5, 9, 17, 33, 65, 100, 1025] { @@ -680,7 +697,7 @@ mod tests { /// The regression this guards: crossing a power-of-two boundary used to /// double the hash count, because the padding leaves were materialized and /// hashed as if they held data. - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_cost_is_linear_across_power_of_two_boundary() { let count_for = |n: usize| { @@ -699,22 +716,22 @@ mod tests { ); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_empty_with_limit_is_the_precomputed_zero_hash() { for (depth, zero_hash) in ZERO_HASHES.iter().enumerate().take(8) { let limit = 1usize << depth; - assert_eq!(merkleize(&Sha2Hasher, &[], Some(limit)), *zero_hash); + assert_eq!(merkleize(&KeccakHasher, &[], Some(limit)), *zero_hash); } // Non-power-of-two limits round up to the enclosing subtree. - assert_eq!(merkleize(&Sha2Hasher, &[], Some(3)), ZERO_HASHES[2]); - assert_eq!(merkleize(&Sha2Hasher, &[], Some(100)), ZERO_HASHES[7]); + assert_eq!(merkleize(&KeccakHasher, &[], Some(3)), ZERO_HASHES[2]); + assert_eq!(merkleize(&KeccakHasher, &[], Some(100)), ZERO_HASHES[7]); } // ── merkleize_progressive tests ── /// Decode a 64-character hex string into a `Node`. - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] fn node_from_hex(s: &str) -> Node { let bytes = s.as_bytes(); assert_eq!(bytes.len(), 64, "expected 64 hex characters"); @@ -730,31 +747,31 @@ mod tests { } /// `hash_tree_root` of a `ProgressiveList[uint64]` holding `values`. - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] fn progressive_u64_list_root(values: &[u64]) -> Node { let mut serialized = Vec::with_capacity(values.len() * 8); for value in values { serialized.extend_from_slice(&value.to_le_bytes()); } - let root = merkleize_progressive(&Sha2Hasher, &pack(&serialized)); - mix_in_length(&Sha2Hasher, &root, values.len()) + let root = merkleize_progressive(&KeccakHasher, &pack(&serialized)); + mix_in_length(&KeccakHasher, &root, values.len()) } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_progressive_empty_is_zero_node() { - assert_eq!(merkleize_progressive(&Sha2Hasher, &[]), [0u8; 32]); + assert_eq!(merkleize_progressive(&KeccakHasher, &[]), [0u8; 32]); } /// EIP-7916 puts the fixed-size subtree on the left and the recursive /// remainder on the right, so a lone chunk hashes as `hash(chunk, zero)`. - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn merkleize_progressive_puts_subtree_on_the_left() { let chunk = [0x11u8; 32]; assert_eq!( - merkleize_progressive(&Sha2Hasher, &[chunk]), - hash_nodes(&Sha2Hasher, &chunk, &[0u8; 32]) + merkleize_progressive(&KeccakHasher, &[chunk]), + hash_nodes(&KeccakHasher, &chunk, &[0u8; 32]) ); } @@ -763,8 +780,9 @@ mod tests { /// sizes pack into 0, 1, 1, 2 and 6 chunks: `n = 5` spans the first two /// subtrees (leaf counts 1 then 4) and `n = 21` reaches the third (16), /// so the larger two exercise the recursion rather than the base case. - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] + #[ignore = "reference roots come from remerkleable, which hashes with SHA-256"] fn merkleize_progressive_matches_reference_roots() { let cases: [(u64, &str); 5] = [ ( @@ -800,8 +818,9 @@ mod tests { /// Roots pinned from the consensus-specs `basic_progressive_list` vectors /// (v1.7.0-alpha.13, `tests/general/phase0/ssz_generic`). - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] + #[ignore = "consensus-spec roots are SHA-256"] fn merkleize_progressive_matches_consensus_spec_vectors() { let cases: [(usize, u64, &str, &str); 5] = [ ( @@ -847,169 +866,169 @@ mod tests { // ── mix_in_length tests ── - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn mix_in_length_known_value() { let root = [0u8; 32]; - let result = mix_in_length(&Sha2Hasher, &root, 5); + let result = mix_in_length(&KeccakHasher, &root, 5); let mut len_node = [0u8; 32]; len_node[..8].copy_from_slice(&5u64.to_le_bytes()); - assert_eq!(result, hash_nodes(&Sha2Hasher, &root, &len_node)); + assert_eq!(result, hash_nodes(&KeccakHasher, &root, &len_node)); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn mix_in_length_zero() { let root = [0xAB; 32]; - let result = mix_in_length(&Sha2Hasher, &root, 0); + let result = mix_in_length(&KeccakHasher, &root, 0); // length=0 means length_node is all zeros - assert_eq!(result, hash_nodes(&Sha2Hasher, &root, &[0u8; 32])); + assert_eq!(result, hash_nodes(&KeccakHasher, &root, &[0u8; 32])); } // ── mix_in_selector tests ── - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn mix_in_selector_known_value() { let root = [0u8; 32]; - let result = mix_in_selector(&Sha2Hasher, &root, 3); + let result = mix_in_selector(&KeccakHasher, &root, 3); let mut sel_node = [0u8; 32]; sel_node[0] = 3; - assert_eq!(result, hash_nodes(&Sha2Hasher, &root, &sel_node)); + assert_eq!(result, hash_nodes(&KeccakHasher, &root, &sel_node)); } // ── HashTreeRoot tests ── - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_bool_false() { - let result = false.hash_tree_root(&Sha2Hasher); + let result = false.hash_tree_root(&KeccakHasher); assert_eq!(result, [0u8; 32]); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_bool_true() { - let result = true.hash_tree_root(&Sha2Hasher); + let result = true.hash_tree_root(&KeccakHasher); let mut expected = [0u8; 32]; expected[0] = 1; assert_eq!(result, expected); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_u64() { let val: u64 = 0x0102030405060708; - let result = val.hash_tree_root(&Sha2Hasher); + let result = val.hash_tree_root(&KeccakHasher); let mut expected = [0u8; 32]; expected[..8].copy_from_slice(&val.to_le_bytes()); assert_eq!(result, expected); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_u8() { let val: u8 = 42; - let result = val.hash_tree_root(&Sha2Hasher); + let result = val.hash_tree_root(&KeccakHasher); let mut expected = [0u8; 32]; expected[0] = 42; assert_eq!(result, expected); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_u128() { let val: u128 = 1; - let result = val.hash_tree_root(&Sha2Hasher); + let result = val.hash_tree_root(&KeccakHasher); let mut expected = [0u8; 32]; expected[..16].copy_from_slice(&val.to_le_bytes()); assert_eq!(result, expected); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_byte_array_32_is_identity() { let arr = [0xab_u8; 32]; - assert_eq!(arr.hash_tree_root(&Sha2Hasher), arr); + assert_eq!(arr.hash_tree_root(&KeccakHasher), arr); assert!(<[u8; 32]>::is_basic_type()); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_byte_array_20_is_padded() { let arr = [0xcd_u8; 20]; - let root = arr.hash_tree_root(&Sha2Hasher); + let root = arr.hash_tree_root(&KeccakHasher); assert_eq!(&root[..20], &arr[..]); assert_eq!(&root[20..], &[0u8; 12]); assert!(<[u8; 20]>::is_basic_type()); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_byte_array_4_is_padded() { let arr = [0x01, 0x02, 0x03, 0x04]; - let root = arr.hash_tree_root(&Sha2Hasher); + let root = arr.hash_tree_root(&KeccakHasher); assert_eq!(&root[..4], &arr[..]); assert_eq!(&root[4..], &[0u8; 28]); assert!(<[u8; 4]>::is_basic_type()); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_byte_array_7_is_padded() { let arr = [0xff_u8; 7]; - let root = arr.hash_tree_root(&Sha2Hasher); + let root = arr.hash_tree_root(&KeccakHasher); assert_eq!(&root[..7], &arr[..]); assert_eq!(&root[7..], &[0u8; 25]); assert!(<[u8; 7]>::is_basic_type()); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_byte_array_48_is_merkleized() { let arr = [0xab_u8; 48]; - let root = arr.hash_tree_root(&Sha2Hasher); + let root = arr.hash_tree_root(&KeccakHasher); // 48 bytes = 2 chunks (32 + 16 zero-padded), merkleize - let expected = merkleize(&Sha2Hasher, &pack(&arr), None); + let expected = merkleize(&KeccakHasher, &pack(&arr), None); assert_eq!(root, expected); assert!(!<[u8; 48]>::is_basic_type()); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_byte_array_52_is_merkleized() { let arr = [0xcd_u8; 52]; - let root = arr.hash_tree_root(&Sha2Hasher); - let expected = merkleize(&Sha2Hasher, &pack(&arr), None); + let root = arr.hash_tree_root(&KeccakHasher); + let expected = merkleize(&KeccakHasher, &pack(&arr), None); assert_eq!(root, expected); assert!(!<[u8; 52]>::is_basic_type()); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_byte_array_96_is_merkleized() { let arr = [0xff_u8; 96]; - let root = arr.hash_tree_root(&Sha2Hasher); - let expected = merkleize(&Sha2Hasher, &pack(&arr), None); + let root = arr.hash_tree_root(&KeccakHasher); + let expected = merkleize(&KeccakHasher, &pack(&arr), None); assert_eq!(root, expected); assert!(!<[u8; 96]>::is_basic_type()); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_vec_u64_empty() { let val: Vec = vec![]; - let result = val.hash_tree_root(&Sha2Hasher); + let result = val.hash_tree_root(&KeccakHasher); // Empty list: merkleize([zero]) then mix_in_length with 0 - let root = merkleize(&Sha2Hasher, &[ZERO_HASHES[0]], None); - assert_eq!(result, mix_in_length(&Sha2Hasher, &root, 0)); + let root = merkleize(&KeccakHasher, &[ZERO_HASHES[0]], None); + assert_eq!(result, mix_in_length(&KeccakHasher, &root, 0)); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_vec_u64() { let val: Vec = vec![1, 2, 3, 4]; - let result = val.hash_tree_root(&Sha2Hasher); + let result = val.hash_tree_root(&KeccakHasher); // 4 u64s = 32 bytes = 1 chunk let mut serialized = Vec::new(); @@ -1017,15 +1036,15 @@ mod tests { serialized.extend_from_slice(&v.to_le_bytes()); } let chunks = pack(&serialized); - let root = merkleize(&Sha2Hasher, &chunks, None); - assert_eq!(result, mix_in_length(&Sha2Hasher, &root, 4)); + let root = merkleize(&KeccakHasher, &chunks, None); + assert_eq!(result, mix_in_length(&KeccakHasher, &root, 4)); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn hash_tree_root_vec_u64_five_elements() { let val: Vec = vec![1, 2, 3, 4, 5]; - let result = val.hash_tree_root(&Sha2Hasher); + let result = val.hash_tree_root(&KeccakHasher); // 5 u64s = 40 bytes = 2 chunks let mut serialized = Vec::new(); @@ -1034,15 +1053,15 @@ mod tests { } let chunks = pack(&serialized); assert_eq!(chunks.len(), 2); - let root = merkleize(&Sha2Hasher, &chunks, None); - assert_eq!(result, mix_in_length(&Sha2Hasher, &root, 5)); + let root = merkleize(&KeccakHasher, &chunks, None); + assert_eq!(result, mix_in_length(&KeccakHasher, &root, 5)); } - #[cfg(feature = "sha2-backend")] + #[cfg(feature = "keccak-backend")] #[test] fn custom_hasher_works() { struct MockHasher; - impl Sha256Hasher for MockHasher { + impl MerkleHasher for MockHasher { fn hash(&self, data: &[u8]) -> [u8; 32] { let mut out = [0u8; 32]; for (i, &b) in data.iter().enumerate() { @@ -1052,15 +1071,15 @@ mod tests { } } let val: u64 = 42; - let root_sha2 = val.hash_tree_root(&Sha2Hasher); + let root_keccak = val.hash_tree_root(&KeccakHasher); let root_mock = val.hash_tree_root(&MockHasher); // u64 HTR doesn't use hashing (just pads to 32 bytes), so both should be equal - assert_eq!(root_sha2, root_mock); + assert_eq!(root_keccak, root_mock); // For a type that DOES hash, they should differ let arr = [0xab_u8; 48]; - let root_sha2 = arr.hash_tree_root(&Sha2Hasher); + let root_keccak = arr.hash_tree_root(&KeccakHasher); let root_mock = arr.hash_tree_root(&MockHasher); - assert_ne!(root_sha2, root_mock); + assert_ne!(root_keccak, root_mock); } } diff --git a/crates/ssz-types/src/hash_tree_root.rs b/crates/ssz-types/src/hash_tree_root.rs index 26748f4..226a933 100644 --- a/crates/ssz-types/src/hash_tree_root.rs +++ b/crates/ssz-types/src/hash_tree_root.rs @@ -2,7 +2,7 @@ use alloc::vec::Vec; use libssz::SszEncode; -use libssz_merkle::{merkleize, mix_in_length, pack, pack_bits, HashTreeRoot, Node, Sha256Hasher}; +use libssz_merkle::{merkleize, mix_in_length, pack, pack_bits, HashTreeRoot, MerkleHasher, Node}; use crate::bitlist::SszBitlist; use crate::bitvector::SszBitvector; @@ -13,7 +13,7 @@ use crate::vector::SszVector; // Vector is a fixed-length composite. HTR = merkleize(field_roots) with no mix_in_length. impl HashTreeRoot for SszVector { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { if T::is_basic_type() { // Basic type vector: pack serialized bytes, merkleize with limit = ceil(N * size / 32) let mut serialized = Vec::new(); @@ -38,7 +38,7 @@ impl HashTreeRoot for SszVector HashTreeRoot for SszList { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { let length = self.len(); if T::is_basic_type() { // Basic type list: pack serialized bytes @@ -66,7 +66,7 @@ impl HashTreeRoot for SszList // Bitvector is fixed-length. HTR = merkleize(pack_bits(bytes, N)). impl HashTreeRoot for SszBitvector { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { let chunks = pack_bits(self.as_bytes(), N); let max_chunks = N.div_ceil(256); // 256 bits per chunk merkleize(hasher, &chunks, Some(max_chunks)) @@ -77,7 +77,7 @@ impl HashTreeRoot for SszBitvector { // Bitlist is variable-length. HTR = mix_in_length(merkleize(pack_bits(bytes, len), limit), len). impl HashTreeRoot for SszBitlist { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { let length = self.len(); let chunks = pack_bits(self.as_bytes(), length); let max_chunks = N.div_ceil(256); @@ -89,14 +89,14 @@ impl HashTreeRoot for SszBitlist { #[cfg(test)] mod tests { use super::*; - use libssz_merkle::Sha2Hasher; + use libssz_merkle::KeccakHasher; // ── SszVector ── #[test] fn vector_u64_hash_tree_root() { let v: SszVector = vec![1u64, 2, 3, 4].try_into().unwrap(); - let root = v.hash_tree_root(&Sha2Hasher); + let root = v.hash_tree_root(&KeccakHasher); // Manual: 4 u64s = 32 bytes = 1 chunk, max_chunks = ceil(4*8/32) = 1 let mut serialized = Vec::new(); @@ -104,14 +104,14 @@ mod tests { val.ssz_append(&mut serialized); } let chunks = pack(&serialized); - let expected = merkleize(&Sha2Hasher, &chunks, Some(1)); + let expected = merkleize(&KeccakHasher, &chunks, Some(1)); assert_eq!(root, expected); } #[test] fn vector_u64_five_elements() { let v: SszVector = vec![1u64, 2, 3, 4, 5].try_into().unwrap(); - let root = v.hash_tree_root(&Sha2Hasher); + let root = v.hash_tree_root(&KeccakHasher); // 5 u64s = 40 bytes = 2 chunks, max_chunks = ceil(5*8/32) = 2 let mut serialized = Vec::new(); @@ -119,7 +119,7 @@ mod tests { val.ssz_append(&mut serialized); } let chunks = pack(&serialized); - let expected = merkleize(&Sha2Hasher, &chunks, Some(2)); + let expected = merkleize(&KeccakHasher, &chunks, Some(2)); assert_eq!(root, expected); } @@ -131,7 +131,7 @@ mod tests { list.push(1).unwrap(); list.push(2).unwrap(); list.push(3).unwrap(); - let root = list.hash_tree_root(&Sha2Hasher); + let root = list.hash_tree_root(&KeccakHasher); // 3 u64s = 24 bytes = 1 chunk, max_chunks = ceil(8*8/32) = 2 let mut serialized = Vec::new(); @@ -139,19 +139,19 @@ mod tests { val.ssz_append(&mut serialized); } let chunks = pack(&serialized); - let inner_root = merkleize(&Sha2Hasher, &chunks, Some(2)); - let expected = mix_in_length(&Sha2Hasher, &inner_root, 3); + let inner_root = merkleize(&KeccakHasher, &chunks, Some(2)); + let expected = mix_in_length(&KeccakHasher, &inner_root, 3); assert_eq!(root, expected); } #[test] fn list_empty_hash_tree_root() { let list: SszList = SszList::new(); - let root = list.hash_tree_root(&Sha2Hasher); + let root = list.hash_tree_root(&KeccakHasher); // Empty: merkleize([], limit=2) then mix_in_length with 0 - let inner_root = merkleize(&Sha2Hasher, &[], Some(2)); - let expected = mix_in_length(&Sha2Hasher, &inner_root, 0); + let inner_root = merkleize(&KeccakHasher, &[], Some(2)); + let expected = mix_in_length(&KeccakHasher, &inner_root, 0); assert_eq!(root, expected); } @@ -162,22 +162,22 @@ mod tests { let mut bv = SszBitvector::<8>::new(); bv.set(0, true).unwrap(); bv.set(7, true).unwrap(); - let root = bv.hash_tree_root(&Sha2Hasher); + let root = bv.hash_tree_root(&KeccakHasher); // 8 bits = 1 byte, pack_bits produces 1 chunk let chunks = pack_bits(bv.as_bytes(), 8); - let expected = merkleize(&Sha2Hasher, &chunks, Some(1)); + let expected = merkleize(&KeccakHasher, &chunks, Some(1)); assert_eq!(root, expected); } #[test] fn bitvector_256_bits() { let bv = SszBitvector::<256>::new(); - let root = bv.hash_tree_root(&Sha2Hasher); + let root = bv.hash_tree_root(&KeccakHasher); // 256 bits = 32 bytes = 1 chunk, max_chunks = 1 let chunks = pack_bits(bv.as_bytes(), 256); - let expected = merkleize(&Sha2Hasher, &chunks, Some(1)); + let expected = merkleize(&KeccakHasher, &chunks, Some(1)); assert_eq!(root, expected); } @@ -189,23 +189,23 @@ mod tests { bl.push(true).unwrap(); bl.push(false).unwrap(); bl.push(true).unwrap(); - let root = bl.hash_tree_root(&Sha2Hasher); + let root = bl.hash_tree_root(&KeccakHasher); // 3 bits, pack_bits produces 1 chunk, max_chunks = ceil(64/256) = 1 let chunks = pack_bits(bl.as_bytes(), 3); - let inner_root = merkleize(&Sha2Hasher, &chunks, Some(1)); - let expected = mix_in_length(&Sha2Hasher, &inner_root, 3); + let inner_root = merkleize(&KeccakHasher, &chunks, Some(1)); + let expected = mix_in_length(&KeccakHasher, &inner_root, 3); assert_eq!(root, expected); } #[test] fn bitlist_empty_hash_tree_root() { let bl = SszBitlist::<64>::new(); - let root = bl.hash_tree_root(&Sha2Hasher); + let root = bl.hash_tree_root(&KeccakHasher); let chunks = pack_bits(bl.as_bytes(), 0); - let inner_root = merkleize(&Sha2Hasher, &chunks, Some(1)); - let expected = mix_in_length(&Sha2Hasher, &inner_root, 0); + let inner_root = merkleize(&KeccakHasher, &chunks, Some(1)); + let expected = mix_in_length(&KeccakHasher, &inner_root, 0); assert_eq!(root, expected); } @@ -220,14 +220,14 @@ mod tests { vec![6u8].try_into().unwrap(), ]; let v: SszVector, 3> = inner.try_into().unwrap(); - let root = v.hash_tree_root(&Sha2Hasher); + let root = v.hash_tree_root(&KeccakHasher); // Composite path: collect roots and merkleize with limit = N let roots: Vec = v .iter() - .map(|item| item.hash_tree_root(&Sha2Hasher)) + .map(|item| item.hash_tree_root(&KeccakHasher)) .collect(); - let expected = merkleize(&Sha2Hasher, &roots, Some(3)); + let expected = merkleize(&KeccakHasher, &roots, Some(3)); assert_eq!(root, expected); } @@ -237,15 +237,15 @@ mod tests { let mut list: SszList, 5> = SszList::new(); list.push(vec![1u8, 2].try_into().unwrap()).unwrap(); list.push(vec![3u8, 4, 5].try_into().unwrap()).unwrap(); - let root = list.hash_tree_root(&Sha2Hasher); + let root = list.hash_tree_root(&KeccakHasher); // Composite path: collect roots, merkleize with limit = N, mix_in_length let roots: Vec = list .iter() - .map(|item| item.hash_tree_root(&Sha2Hasher)) + .map(|item| item.hash_tree_root(&KeccakHasher)) .collect(); - let inner_root = merkleize(&Sha2Hasher, &roots, Some(5)); - let expected = mix_in_length(&Sha2Hasher, &inner_root, 2); + let inner_root = merkleize(&KeccakHasher, &roots, Some(5)); + let expected = mix_in_length(&KeccakHasher, &inner_root, 2); assert_eq!(root, expected); } } diff --git a/crates/ssz-types/src/progressive_bitlist.rs b/crates/ssz-types/src/progressive_bitlist.rs index d2c8908..3aca812 100644 --- a/crates/ssz-types/src/progressive_bitlist.rs +++ b/crates/ssz-types/src/progressive_bitlist.rs @@ -4,7 +4,7 @@ use smallvec::SmallVec; use crate::error::IndexError; use libssz_merkle::{ - merkleize_progressive, mix_in_length, pack_bits, HashTreeRoot, Node, Sha256Hasher, + merkleize_progressive, mix_in_length, pack_bits, HashTreeRoot, MerkleHasher, Node, }; /// A progressive bitlist: ordered variable-length collection of booleans **without limit**. @@ -173,7 +173,7 @@ impl SszDecode for ProgressiveBitlist { // mix_in_length(merkleize_progressive(pack_bits(bytes, len)), len) impl HashTreeRoot for ProgressiveBitlist { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { let length = self.len; let chunks = pack_bits(self.as_bytes(), length); let root = merkleize_progressive(hasher, &chunks); diff --git a/crates/ssz-types/src/progressive_list.rs b/crates/ssz-types/src/progressive_list.rs index b909011..108a917 100644 --- a/crates/ssz-types/src/progressive_list.rs +++ b/crates/ssz-types/src/progressive_list.rs @@ -1,7 +1,7 @@ use alloc::vec::Vec; use core::ops::Deref; use libssz::{DecodeError, SszDecode, SszEncode}; -use libssz_merkle::{merkleize_progressive, mix_in_length, pack, HashTreeRoot, Node, Sha256Hasher}; +use libssz_merkle::{merkleize_progressive, mix_in_length, pack, HashTreeRoot, MerkleHasher, Node}; /// A progressive list: ordered variable-length homogeneous collection **without limit**. /// @@ -110,7 +110,7 @@ impl SszDecode for ProgressiveList { // mix_in_length(merkleize_progressive(chunks), len) impl HashTreeRoot for ProgressiveList { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { let length = self.len(); let chunks: Vec = if T::is_basic_type() { diff --git a/docs/technical-design.md b/docs/technical-design.md index d1bc16e..f8f4309 100644 --- a/docs/technical-design.md +++ b/docs/technical-design.md @@ -156,7 +156,7 @@ Offsets are validated at multiple levels: ## Merkleization algorithm -SSZ Merkleization converts any value into a 32-byte Merkle root using SHA-256. +SSZ Merkleization converts any value into a 32-byte Merkle root. This library hashes with Keccak-256 (the consensus spec uses SHA-256, so roots are not consensus-compatible). ### Basic types @@ -202,7 +202,7 @@ merkleize(chunks, limit): ``` ZERO_HASHES[0] = [0u8; 32] -ZERO_HASHES[i] = SHA256(ZERO_HASHES[i-1] || ZERO_HASHES[i-1]) +ZERO_HASHES[i] = Keccak256(ZERO_HASHES[i-1] || ZERO_HASHES[i-1]) ``` Build-time computation avoids both runtime cost and `lazy_static` / `once_cell` dependencies. @@ -225,7 +225,7 @@ All trait methods on basic types are `#[inline(always)]`. For primitives like `u ### Precomputed zero hashes -Build-time computation of `ZERO_HASHES` eliminates 64 SHA-256 hash calls from the hot path. The table is a static array with no initialization cost. +Build-time computation of `ZERO_HASHES` eliminates 64 Keccak-256 hash calls from the hot path. The table is a static array with no initialization cost. ## no_std strategy @@ -298,4 +298,4 @@ Errors are data, not strings. Each variant carries the numeric values needed to | Derive macros | Separate crate | Separate crate | Integrated | | Zero hashes | Build-time precomputed | `lazy_static` | Computed at init | | Error types | Flat enum | Flat enum | Flat enum | -| Hash function | sha2 crate | ring / sha2 | sha2 | +| Hash function | sha3 crate (Keccak-256) | ring / sha2 | sha2 | diff --git a/fuzz/fuzz_targets/diff_hash_tree_root.rs b/fuzz/fuzz_targets/diff_hash_tree_root.rs index 46182ad..06665be 100644 --- a/fuzz/fuzz_targets/diff_hash_tree_root.rs +++ b/fuzz/fuzz_targets/diff_hash_tree_root.rs @@ -74,7 +74,7 @@ struct FuzzInput { /// Hash with our library. fn ours(val: &T) -> [u8; 32] { - val.hash_tree_root(&libssz_merkle::Sha2Hasher) + val.hash_tree_root(&libssz_merkle::KeccakHasher) } /// Hash with Lighthouse's tree_hash. diff --git a/fuzz/fuzz_targets/diff_htr_container.rs b/fuzz/fuzz_targets/diff_htr_container.rs index f9a737b..2790796 100644 --- a/fuzz/fuzz_targets/diff_htr_container.rs +++ b/fuzz/fuzz_targets/diff_htr_container.rs @@ -88,7 +88,7 @@ struct FuzzInput { /// Hash with our library. fn ours(val: &T) -> [u8; 32] { - val.hash_tree_root(&libssz_merkle::Sha2Hasher) + val.hash_tree_root(&libssz_merkle::KeccakHasher) } /// Hash with Lighthouse's tree_hash. diff --git a/fuzz/fuzz_targets/hash_tree_root_fuzz.rs b/fuzz/fuzz_targets/hash_tree_root_fuzz.rs index 1b4115d..7f59c5d 100644 --- a/fuzz/fuzz_targets/hash_tree_root_fuzz.rs +++ b/fuzz/fuzz_targets/hash_tree_root_fuzz.rs @@ -33,9 +33,9 @@ struct FuzzInput { fuzz_target!(|input: FuzzInput| { // Primitives - let _ = input.val_u64.hash_tree_root(&libssz_merkle::Sha2Hasher); - let _ = input.val_bool.hash_tree_root(&libssz_merkle::Sha2Hasher); - let _ = input.val_bytes32.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = input.val_u64.hash_tree_root(&libssz_merkle::KeccakHasher); + let _ = input.val_bool.hash_tree_root(&libssz_merkle::KeccakHasher); + let _ = input.val_bytes32.hash_tree_root(&libssz_merkle::KeccakHasher); // Container let header = BeaconBlockHeader { @@ -45,65 +45,65 @@ fuzz_target!(|input: FuzzInput| { state_root: input.state_root, body_root: input.body_root, }; - let _ = header.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = header.hash_tree_root(&libssz_merkle::KeccakHasher); // Decode-then-hash from raw bytes (cap sizes for performance) if let Ok(val) = Vec::::from_ssz_bytes(&input.raw) { if val.len() <= 4096 { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } } if let Ok(val) = SszList::::from_ssz_bytes(&input.raw) { if val.len() <= 4096 { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } } if let Ok(val) = SszVector::<[u8; 32], 64>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } if let Ok(val) = SszBitlist::<2048>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } if let Ok(val) = SszBitvector::<512>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } // -- Additional bitfield sizes -- if let Ok(val) = SszBitlist::<8>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } if let Ok(val) = SszBitlist::<64>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } if let Ok(val) = SszBitvector::<8>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } if let Ok(val) = SszBitvector::<64>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } // -- Non-byte-aligned bitvector sizes for excess bit validation -- if let Ok(val) = SszBitvector::<3>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } if let Ok(val) = SszBitvector::<7>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } if let Ok(val) = SszBitvector::<9>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } if let Ok(val) = SszBitvector::<15>::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } // -- Boundary collections -- if let Ok(val) = SszVector::::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } // -- Empty collection HTR -- - let _ = Vec::::new().hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = Vec::::new().hash_tree_root(&libssz_merkle::KeccakHasher); if let Ok(bl) = SszBitlist::<2048>::from_ssz_bytes(&[1u8]) { - let _ = bl.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = bl.hash_tree_root(&libssz_merkle::KeccakHasher); } }); diff --git a/fuzz/fuzz_targets/merkle_primitives.rs b/fuzz/fuzz_targets/merkle_primitives.rs index b56ddf8..885fb95 100644 --- a/fuzz/fuzz_targets/merkle_primitives.rs +++ b/fuzz/fuzz_targets/merkle_primitives.rs @@ -5,10 +5,10 @@ use arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; -use libssz_merkle::{hash_nodes, merkleize, mix_in_length, pack, pack_bits, Node, Sha2Hasher}; +use libssz_merkle::{hash_nodes, merkleize, mix_in_length, pack, pack_bits, Node, KeccakHasher}; -fn h() -> Sha2Hasher { - Sha2Hasher +fn h() -> KeccakHasher { + KeccakHasher } #[derive(Debug, Arbitrary)] diff --git a/fuzz/fuzz_targets/union_htr.rs b/fuzz/fuzz_targets/union_htr.rs index 664b723..330699d 100644 --- a/fuzz/fuzz_targets/union_htr.rs +++ b/fuzz/fuzz_targets/union_htr.rs @@ -37,26 +37,26 @@ fuzz_target!(|input: FuzzInput| { let bytes32_variant = TestUnion::Bytes32Val(input.bytes32_val); let vec_variant = TestUnion::VecU8Val(vec_u8); - let _ = u64_variant.hash_tree_root(&libssz_merkle::Sha2Hasher); - let _ = bool_variant.hash_tree_root(&libssz_merkle::Sha2Hasher); - let _ = bytes32_variant.hash_tree_root(&libssz_merkle::Sha2Hasher); - let _ = vec_variant.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = u64_variant.hash_tree_root(&libssz_merkle::KeccakHasher); + let _ = bool_variant.hash_tree_root(&libssz_merkle::KeccakHasher); + let _ = bytes32_variant.hash_tree_root(&libssz_merkle::KeccakHasher); + let _ = vec_variant.hash_tree_root(&libssz_merkle::KeccakHasher); // Verify determinism: same input → same root - let root1 = TestUnion::U64Val(input.u64_val).hash_tree_root(&libssz_merkle::Sha2Hasher); - let root2 = TestUnion::U64Val(input.u64_val).hash_tree_root(&libssz_merkle::Sha2Hasher); + let root1 = TestUnion::U64Val(input.u64_val).hash_tree_root(&libssz_merkle::KeccakHasher); + let root2 = TestUnion::U64Val(input.u64_val).hash_tree_root(&libssz_merkle::KeccakHasher); assert_eq!(root1, root2, "U64Val HTR not deterministic"); - let root1 = TestUnion::BoolVal(input.bool_val).hash_tree_root(&libssz_merkle::Sha2Hasher); - let root2 = TestUnion::BoolVal(input.bool_val).hash_tree_root(&libssz_merkle::Sha2Hasher); + let root1 = TestUnion::BoolVal(input.bool_val).hash_tree_root(&libssz_merkle::KeccakHasher); + let root2 = TestUnion::BoolVal(input.bool_val).hash_tree_root(&libssz_merkle::KeccakHasher); assert_eq!(root1, root2, "BoolVal HTR not deterministic"); - let root1 = TestUnion::Bytes32Val(input.bytes32_val).hash_tree_root(&libssz_merkle::Sha2Hasher); - let root2 = TestUnion::Bytes32Val(input.bytes32_val).hash_tree_root(&libssz_merkle::Sha2Hasher); + let root1 = TestUnion::Bytes32Val(input.bytes32_val).hash_tree_root(&libssz_merkle::KeccakHasher); + let root2 = TestUnion::Bytes32Val(input.bytes32_val).hash_tree_root(&libssz_merkle::KeccakHasher); assert_eq!(root1, root2, "Bytes32Val HTR not deterministic"); // Decode from raw bytes; if successful, hash_tree_root must not panic if let Ok(val) = TestUnion::from_ssz_bytes(&input.raw) { - let _ = val.hash_tree_root(&libssz_merkle::Sha2Hasher); + let _ = val.hash_tree_root(&libssz_merkle::KeccakHasher); } }); diff --git a/spec-tests/src/types/compatible_unions.rs b/spec-tests/src/types/compatible_unions.rs index fdd130e..9310925 100644 --- a/spec-tests/src/types/compatible_unions.rs +++ b/spec-tests/src/types/compatible_unions.rs @@ -1,5 +1,5 @@ use libssz::{DecodeError, SszDecode, SszEncode}; -use libssz_merkle::{mix_in_selector, HashTreeRoot, Node, Sha256Hasher}; +use libssz_merkle::{mix_in_selector, HashTreeRoot, MerkleHasher, Node}; use libssz_types::split_union_bytes; use super::progressive_containers::{ @@ -56,7 +56,7 @@ impl SszDecode for CompatibleUnionA { } impl HashTreeRoot for CompatibleUnionA { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { match self { Self::V1(v) => mix_in_selector(hasher, &v.hash_tree_root(hasher), 1), } @@ -119,7 +119,7 @@ impl SszDecode for CompatibleUnionBC { } impl HashTreeRoot for CompatibleUnionBC { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { match self { Self::V2(v) => mix_in_selector(hasher, &v.hash_tree_root(hasher), 2), Self::V3(v) => mix_in_selector(hasher, &v.hash_tree_root(hasher), 3), @@ -203,7 +203,7 @@ impl SszDecode for CompatibleUnionABCA { } impl HashTreeRoot for CompatibleUnionABCA { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { match self { Self::V1(v) => mix_in_selector(hasher, &v.hash_tree_root(hasher), 1), Self::V2(v) => mix_in_selector(hasher, &v.hash_tree_root(hasher), 2), diff --git a/spec-tests/src/types/progressive_containers.rs b/spec-tests/src/types/progressive_containers.rs index 53e81f3..c20cab9 100644 --- a/spec-tests/src/types/progressive_containers.rs +++ b/spec-tests/src/types/progressive_containers.rs @@ -1,5 +1,5 @@ use libssz_derive::{HashTreeRoot as HashTreeRootDerive, SszDecode, SszEncode}; -use libssz_merkle::{HashTreeRoot, Node, Sha256Hasher}; +use libssz_merkle::{HashTreeRoot, MerkleHasher, Node}; use libssz_types::{ProgressiveBitlist, ProgressiveList, SszBitlist, SszBitvector, SszList}; use super::containers::{SmallTestStruct, VarTestStruct}; @@ -68,7 +68,7 @@ pub struct ProgressiveTestStruct { } impl HashTreeRoot for ProgressiveTestStruct { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { // Regular container merkleization (not progressive) let field_roots: [Node; 4] = [ self.a.hash_tree_root(hasher), @@ -102,7 +102,7 @@ pub struct ProgressiveBitsStruct { } impl HashTreeRoot for ProgressiveBitsStruct { - fn hash_tree_root(&self, hasher: &impl Sha256Hasher) -> Node { + fn hash_tree_root(&self, hasher: &impl MerkleHasher) -> Node { let field_roots: [Node; 12] = [ self.a.hash_tree_root(hasher), self.b.hash_tree_root(hasher), diff --git a/spec-tests/tests/ssz_generic.rs b/spec-tests/tests/ssz_generic.rs index ddc4c05..57ddfdc 100644 --- a/spec-tests/tests/ssz_generic.rs +++ b/spec-tests/tests/ssz_generic.rs @@ -27,7 +27,7 @@ fn boolean_valid() { assert_eq!(reencoded, ssz, "{case_name}: re-encoded bytes mismatch"); // Hash tree root - let root = decoded.hash_tree_root(&libssz_merkle::Sha2Hasher); + let root = decoded.hash_tree_root(&libssz_merkle::KeccakHasher); assert_eq!(root, expected_root, "{case_name}: hash tree root mismatch"); } } @@ -148,7 +148,7 @@ fn check_uint( let reencoded = decoded.to_ssz(); assert_eq!(reencoded, ssz, "{case_name}: roundtrip mismatch"); - let root = decoded.hash_tree_root(&libssz_merkle::Sha2Hasher); + let root = decoded.hash_tree_root(&libssz_merkle::KeccakHasher); assert_eq!(&root, expected_root, "{case_name}: hash tree root mismatch"); } @@ -169,7 +169,7 @@ fn check_uint_256(ssz: &[u8], yaml_str: &str, expected_root: &[u8; 32], case_nam // Hash tree root: for a basic type of 32 bytes, the root IS the value // (already chunk-sized, no padding needed) - let root = decoded.hash_tree_root(&libssz_merkle::Sha2Hasher); + let root = decoded.hash_tree_root(&libssz_merkle::KeccakHasher); assert_eq!(&root, expected_root, "{case_name}: hash tree root mismatch"); } @@ -260,7 +260,7 @@ fn basic_vector_valid() { .unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + decoded.hash_tree_root(&libssz_merkle::KeccakHasher), *expected_root, "{case_name}: root" ); @@ -307,7 +307,7 @@ fn basic_vector_valid() { .unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + decoded.hash_tree_root(&libssz_merkle::KeccakHasher), *expected_root, "{case_name}: root" ); @@ -354,7 +354,7 @@ fn basic_vector_valid() { .unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + decoded.hash_tree_root(&libssz_merkle::KeccakHasher), *expected_root, "{case_name}: root" ); @@ -401,7 +401,7 @@ fn basic_vector_valid() { .unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + decoded.hash_tree_root(&libssz_merkle::KeccakHasher), *expected_root, "{case_name}: root" ); @@ -448,7 +448,7 @@ fn basic_vector_valid() { .unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + decoded.hash_tree_root(&libssz_merkle::KeccakHasher), *expected_root, "{case_name}: root" ); @@ -495,7 +495,7 @@ fn basic_vector_valid() { .unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + decoded.hash_tree_root(&libssz_merkle::KeccakHasher), *expected_root, "{case_name}: root" ); @@ -542,7 +542,7 @@ fn basic_vector_valid() { .unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + decoded.hash_tree_root(&libssz_merkle::KeccakHasher), *expected_root, "{case_name}: root" ); @@ -909,7 +909,7 @@ fn bitlist_valid() { .unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + decoded.hash_tree_root(&libssz_merkle::KeccakHasher), *expected_root, "{case_name}: root" ); @@ -1018,7 +1018,7 @@ fn bitvector_valid() { .unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + decoded.hash_tree_root(&libssz_merkle::KeccakHasher), *expected_root, "{case_name}: root" ); @@ -1185,7 +1185,7 @@ fn check_container( T::from_ssz_bytes(ssz).unwrap_or_else(|e| panic!("{case_name}: decode failed: {e:?}")); assert_eq!(decoded.to_ssz(), ssz, "{case_name}: roundtrip"); assert_eq!( - &decoded.hash_tree_root(&libssz_merkle::Sha2Hasher), + &decoded.hash_tree_root(&libssz_merkle::KeccakHasher), expected_root, "{case_name}: root" ); @@ -1275,7 +1275,7 @@ fn check_roundtrip_root Date: Thu, 12 Nov 2026 10:59:45 -0300 Subject: [PATCH 2/5] perf: batch merkleize levels and speed up SHA-256 pair hashing - `Sha256Hasher::hash_pairs` hashes many 64-byte pairs per call. It has a default implementation, so existing hashers keep working unchanged. `merkleize` hands each tree level to it in one batch, and `hash_nodes` routes single pairs through it. - `merkleize` hashes level 0 straight from the input chunks instead of copying them first. - `Sha2Hasher` moves to `sha2` 0.11, which selects ARMv8 SHA2 instructions at runtime (0.10 only used them with its `asm` feature), and hashes pairs by calling the compression function directly on the message plus a constant padding block. - New opt-in `hashtree` feature: pairs are hashed by the hashtree assembly library (SHA-NI, AVX-512/AVX2 multi-buffer, ARMv8 SHA2, NEON). Requires a C toolchain; CI tests it as a separate matrix entry. --- .github/workflows/ci.yml | 2 + crates/ssz-merkle/Cargo.toml | 8 +- crates/ssz-merkle/src/lib.rs | 96 +++++++++++++--------- crates/ssz-merkle/src/sha256.rs | 138 ++++++++++++++++++++++++++++++++ docs/technical-design.md | 4 +- 5 files changed, 208 insertions(+), 40 deletions(-) create mode 100644 crates/ssz-merkle/src/sha256.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5258ce7..b91cbae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,6 +80,8 @@ jobs: args: "" - features: alloc-only args: "--no-default-features --features alloc" + - features: hashtree + args: "--features libssz-merkle/hashtree" steps: - uses: actions/checkout@v7 - uses: actions/cache@v6 diff --git a/crates/ssz-merkle/Cargo.toml b/crates/ssz-merkle/Cargo.toml index cae1346..4c83e59 100644 --- a/crates/ssz-merkle/Cargo.toml +++ b/crates/ssz-merkle/Cargo.toml @@ -14,12 +14,16 @@ default = ["std"] std = ["alloc", "libssz/std", "sha2-backend"] alloc = ["libssz/alloc"] sha2-backend = ["dep:sha2"] +# Hash Merkle pairs with the `hashtree` library: multi-buffer SHA-256 assembly +# specialized for 64-byte inputs. Needs a C toolchain and `make` at build time. +hashtree = ["std", "sha2-backend", "dep:hashtree-rs"] ethereum_types = ["dep:ethereum-types", "libssz/ethereum_types"] [dependencies] libssz = { version = "0.3.0", path = "../ssz", default-features = false } -sha2 = { version = "0.10", default-features = false, optional = true } +sha2 = { version = "0.11", default-features = false, optional = true } +hashtree-rs = { version = "0.2.6", optional = true } ethereum-types = { version = "0.15", default-features = false, optional = true } [build-dependencies] -sha2 = "0.10" +sha2 = "0.11" diff --git a/crates/ssz-merkle/src/lib.rs b/crates/ssz-merkle/src/lib.rs index f175298..558bb42 100644 --- a/crates/ssz-merkle/src/lib.rs +++ b/crates/ssz-merkle/src/lib.rs @@ -16,33 +16,43 @@ pub type Node = [u8; 32]; include!(concat!(env!("OUT_DIR"), "/zero_hashes.rs")); +#[cfg(feature = "sha2-backend")] +mod sha256; +#[cfg(feature = "sha2-backend")] +pub use sha256::Sha2Hasher; + /// Trait for pluggable SHA-256 implementations. /// /// Instance method (`&self`) enables passing `&impl Sha256Hasher` through the /// call chain. The hasher struct is typically zero-sized so `&self` compiles away. pub trait Sha256Hasher { + /// SHA-256 of arbitrary bytes. fn hash(&self, data: &[u8]) -> [u8; 32]; -} -/// Default SHA-256 hasher using the `sha2` crate. -#[cfg(feature = "sha2-backend")] -pub struct Sha2Hasher; - -#[cfg(feature = "sha2-backend")] -impl Sha256Hasher for Sha2Hasher { - fn hash(&self, data: &[u8]) -> [u8; 32] { - use sha2::{Digest, Sha256}; - Sha256::digest(data).into() + /// Hash each pair, `SHA256(pair[0] || pair[1])`, into the matching slot of `out`. + /// + /// This is the hot path of Merkleization: [`merkleize`] hashes a whole tree + /// level per call and [`hash_nodes`] passes a single pair, so an + /// implementation can batch pairs or specialize for 64-byte messages. The + /// default hashes one pair at a time through [`hash`](Self::hash). + /// + /// # Panics + /// + /// Panics if `pairs.len() != out.len()`. + fn hash_pairs(&self, pairs: &[[Node; 2]], out: &mut [Node]) { + assert_eq!(pairs.len(), out.len(), "one output node per pair"); + for (pair, out) in pairs.iter().zip(out) { + *out = self.hash(pair.as_flattened()); + } } } /// Hash two nodes together: SHA256(a || b). #[inline] pub fn hash_nodes(hasher: &impl Sha256Hasher, a: &Node, b: &Node) -> Node { - let mut buf = [0u8; 64]; - buf[..32].copy_from_slice(a); - buf[32..].copy_from_slice(b); - hasher.hash(&buf) + let mut out = [[0u8; 32]]; + hasher.hash_pairs(&[[*a, *b]], &mut out); + out[0] } /// Pack serialized bytes into 32-byte chunks, zero-padding the last chunk if needed. @@ -79,7 +89,9 @@ pub fn pack_bits(bits: &[u8], num_bits: usize) -> Vec { /// all zeros, so its root is read from [`ZERO_HASHES`] instead of being hashed. /// /// Cost is `O(chunks.len())` plus one hash per virtual level above the real subtree. -/// Two buffers are allocated and reused (≈ `chunks.len()` and `chunks.len().div_ceil(2)` nodes). +/// Each tree level is hashed in one [`Sha256Hasher::hash_pairs`] batch. Two buffers are +/// allocated and reused (≈ `chunks.len() / 2` and `chunks.len() / 4` nodes); the leaves are +/// never copied. #[cfg(feature = "alloc")] pub fn merkleize(hasher: &impl Sha256Hasher, chunks: &[Node], limit: Option) -> Node { let count = match limit { @@ -110,36 +122,46 @@ pub fn merkleize(hasher: &impl Sha256Hasher, chunks: &[Node], limit: Option = chunks.to_vec(); - let mut next: Vec = Vec::with_capacity(layer.len().div_ceil(2)); - - for zero_hash in ZERO_HASHES.iter().take(real_depth) { - next.clear(); - let (pairs, remainder) = layer.as_chunks::<2>(); - for pair in pairs { - next.push(hash_nodes(hasher, &pair[0], &pair[1])); + // Fold the real chunks one level at a time. Level 0 reads `chunks` in place; + // later levels swap between two buffers allocated once, so no level allocates. + let mut root = if real_depth == 0 { + chunks[0] + } else { + let mut layer: Vec = Vec::with_capacity(chunks.len().div_ceil(2)); + hash_level(hasher, chunks, &ZERO_HASHES[0], &mut layer); + let mut next: Vec = Vec::with_capacity(layer.len().div_ceil(2)); + for zero_hash in &ZERO_HASHES[1..real_depth] { + hash_level(hasher, &layer, zero_hash, &mut next); + core::mem::swap(&mut layer, &mut next); } - if let [odd] = remainder { - next.push(hash_nodes(hasher, odd, zero_hash)); - } - core::mem::swap(&mut layer, &mut next); - } + layer[0] + }; - // `layer[0]` now roots the real-data subtree. Merge in the virtual levels. - let mut root = layer[0]; - for zero_hash in ZERO_HASHES.iter().take(depth).skip(real_depth) { + // `root` now roots the real-data subtree. Merge in the virtual levels. + for zero_hash in &ZERO_HASHES[real_depth..depth] { root = hash_nodes(hasher, &root, zero_hash); } root } +/// Hash one tree level of `layer` into `next`. +/// +/// Full pairs of real nodes are hashed in a single [`Sha256Hasher::hash_pairs`] +/// batch. A level with an odd node count ends on the frontier, where the missing +/// right sibling roots an all-zero subtree, so `zero_hash` (the precomputed root +/// of that subtree) stands in for it rather than being hashed. +#[cfg(feature = "alloc")] +fn hash_level(hasher: &impl Sha256Hasher, layer: &[Node], zero_hash: &Node, next: &mut Vec) { + let (pairs, remainder) = layer.as_chunks::<2>(); + next.clear(); + next.resize(pairs.len(), [0u8; 32]); + hasher.hash_pairs(pairs, next); + if let [odd] = remainder { + next.push(hash_nodes(hasher, odd, zero_hash)); + } +} + /// Progressive merkleization: a chain of progressively larger binary subtrees. /// /// Creates a 0-terminated sequence of binary subtrees with increasing leaf count: diff --git a/crates/ssz-merkle/src/sha256.rs b/crates/ssz-merkle/src/sha256.rs new file mode 100644 index 0000000..6e276c0 --- /dev/null +++ b/crates/ssz-merkle/src/sha256.rs @@ -0,0 +1,138 @@ +//! Default SHA-256 backend: the `sha2` crate, or `hashtree` with the `hashtree` feature. + +use crate::{Node, Sha256Hasher}; + +/// Default SHA-256 hasher. +/// +/// Merkle pairs are hashed by calling the SHA-256 compression function directly +/// on the 64-byte message and a constant padding block, skipping the +/// general-purpose `Digest` buffering. `sha2` selects CPU SHA instructions at +/// runtime (x86 SHA-NI, ARMv8 SHA2) and falls back to software otherwise. +/// +/// With the `hashtree` feature, pairs are hashed by the +/// [`hashtree`](https://github.com/OffchainLabs/hashtree) library instead: +/// assembly specialized for 64-byte inputs that also hashes several pairs at +/// once (SHA-NI, AVX-512 x16, AVX2 x8, ARMv8 SHA2, NEON x4). +pub struct Sha2Hasher; + +impl Sha256Hasher for Sha2Hasher { + fn hash(&self, data: &[u8]) -> [u8; 32] { + use sha2::{Digest, Sha256}; + Sha256::digest(data).into() + } + + #[cfg(not(feature = "hashtree"))] + fn hash_pairs(&self, pairs: &[[Node; 2]], out: &mut [Node]) { + assert_eq!(pairs.len(), out.len(), "one output node per pair"); + for (pair, out) in pairs.iter().zip(out) { + *out = hash_pair(pair); + } + } + + #[cfg(feature = "hashtree")] + fn hash_pairs(&self, pairs: &[[Node; 2]], out: &mut [Node]) { + // `hashtree_rs::hash` performs no length checks of its own. + assert_eq!(pairs.len(), out.len(), "one output node per pair"); + if pairs.is_empty() { + return; + } + // The C library selects its implementation lazily by writing a static + // function pointer on first use; do that once, before any concurrent call. + static INIT: std::sync::Once = std::sync::Once::new(); + INIT.call_once(|| { + hashtree_rs::init(); + }); + hashtree_rs::hash( + out.as_flattened_mut(), + pairs.as_flattened().as_flattened(), + pairs.len(), + ); + } +} + +/// SHA-256 initial hash value (FIPS 180-4, 5.3.3). +const IV: [u32; 8] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +]; + +/// Padding for a 64-byte message, which always fills a block of its own: +/// `0x80`, zeros, then the message length in bits (512) as a big-endian `u64`. +const PADDING_BLOCK: [u8; 64] = { + let mut block = [0u8; 64]; + block[0] = 0x80; + block[62] = 0x02; + block +}; + +/// SHA-256 of `pair[0] || pair[1]`, as two calls to the compression function. +#[cfg_attr(feature = "hashtree", allow(dead_code))] +#[inline] +fn hash_pair(pair: &[Node; 2]) -> Node { + let mut message = [0u8; 64]; + message[..32].copy_from_slice(&pair[0]); + message[32..].copy_from_slice(&pair[1]); + + let mut state = IV; + sha2::block_api::compress256(&mut state, &[message, PADDING_BLOCK]); + + let mut digest = [0u8; 32]; + let (words, _) = digest.as_chunks_mut::<4>(); + for (bytes, word) in words.iter_mut().zip(state) { + *bytes = word.to_be_bytes(); + } + digest +} + +#[cfg(all(test, feature = "alloc"))] +mod tests { + use super::*; + use alloc::vec::Vec; + use sha2::{Digest, Sha256}; + + /// Pairs whose bytes differ in every position and set high bits, so a + /// misplaced word or byte-order slip changes the digest. + fn sample_pairs(count: usize) -> Vec<[Node; 2]> { + (0..count) + .map(|i| { + let mut pair = [[0u8; 32]; 2]; + for (j, byte) in pair.as_flattened_mut().iter_mut().enumerate() { + *byte = (i * 131 + j * 29 + 0x80) as u8; + } + pair + }) + .collect() + } + + fn digest(pair: &[Node; 2]) -> Node { + Sha256::digest(pair.as_flattened()).into() + } + + #[test] + fn hash_pair_matches_digest() { + for pair in sample_pairs(16) { + assert_eq!(hash_pair(&pair), digest(&pair)); + } + } + + /// Covers the batch path used by `merkleize`, including the `hashtree` + /// backend when that feature is enabled, across every SIMD batch remainder. + #[test] + fn hash_pairs_matches_digest_for_every_count() { + for count in 0..=33 { + let pairs = sample_pairs(count); + let mut out = alloc::vec![[0u8; 32]; count]; + Sha2Hasher.hash_pairs(&pairs, &mut out); + for (i, (pair, node)) in pairs.iter().zip(&out).enumerate() { + assert_eq!(*node, digest(pair), "pair {i} of {count}"); + } + } + } + + #[test] + #[should_panic(expected = "one output node per pair")] + fn hash_pairs_rejects_mismatched_output() { + let pairs = sample_pairs(3); + let mut out = [[0u8; 32]; 2]; + Sha2Hasher.hash_pairs(&pairs, &mut out); + } +} diff --git a/docs/technical-design.md b/docs/technical-design.md index d1bc16e..a417b55 100644 --- a/docs/technical-design.md +++ b/docs/technical-design.md @@ -196,6 +196,8 @@ merkleize(chunks, limit): **Virtual padding:** rather than allocating `limit` chunks, the implementation uses precomputed `ZERO_HASHES[level]` for any subtree that would be all zeros. This keeps memory usage proportional to the actual number of non-zero chunks. +**Batched levels:** every full pair of a level is handed to `Sha256Hasher::hash_pairs` in one call, and `hash_nodes` passes a single pair through the same method. A hasher can therefore hash many 64-byte messages at once (multi-buffer SIMD) or skip general-purpose padding. The default `Sha2Hasher` calls the `sha2` compression function directly on the message and a constant padding block (`sha2` picks SHA-NI / ARMv8 SHA2 at runtime). The optional `hashtree` feature hands each batch to the [hashtree](https://github.com/OffchainLabs/hashtree) assembly library instead, at the cost of a C toolchain at build time. + ### Zero hash table `ZERO_HASHES` is a static array of 65 entries (depth 0 through 64), computed at build time by `ssz-merkle/build.rs`: @@ -298,4 +300,4 @@ Errors are data, not strings. Each variant carries the numeric values needed to | Derive macros | Separate crate | Separate crate | Integrated | | Zero hashes | Build-time precomputed | `lazy_static` | Computed at init | | Error types | Flat enum | Flat enum | Flat enum | -| Hash function | sha2 crate | ring / sha2 | sha2 | +| Hash function | sha2 crate (optional hashtree) | ring / sha2 | sha2 | From 88557bb66dab1b95dd1eb4ff56127bf89e5cd388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:52:50 -0300 Subject: [PATCH 3/5] perf: batch merkleize levels and hash Keccak pairs with SIMD Port the level batching from perf/batched-sha256-htr to `MerkleHasher`: `merkleize` hands every full pair of a level to `hash_pairs` in one call, and `hash_nodes` goes through the same method. `KeccakHasher` moves to its own module and skips the general-purpose sponge: each 64-byte pair is one Keccak-f[1600] permutation of a pre-padded state. `hash_pairs` runs several permutations at once, chosen at runtime with `cpufeatures`: AVX-512 (8 pairs), AVX2 (4) or the ARMv8 SHA3 extension (2). Leftover pairs and other CPUs use the scalar permutation from the `keccak` crate. Each backend is tested against `sha3::Keccak256`. --- crates/ssz-merkle/Cargo.toml | 7 +- crates/ssz-merkle/src/keccak256.rs | 654 +++++++++++++++++++++++++++++ crates/ssz-merkle/src/lib.rs | 99 +++-- docs/technical-design.md | 2 + 4 files changed, 721 insertions(+), 41 deletions(-) create mode 100644 crates/ssz-merkle/src/keccak256.rs diff --git a/crates/ssz-merkle/Cargo.toml b/crates/ssz-merkle/Cargo.toml index b5a7381..58df738 100644 --- a/crates/ssz-merkle/Cargo.toml +++ b/crates/ssz-merkle/Cargo.toml @@ -13,12 +13,17 @@ categories = ["encoding", "cryptography::cryptocurrencies"] default = ["std"] std = ["alloc", "libssz/std", "keccak-backend"] alloc = ["libssz/alloc"] -keccak-backend = ["dep:sha3"] +keccak-backend = ["dep:sha3", "dep:keccak", "dep:cpufeatures"] ethereum_types = ["dep:ethereum-types", "libssz/ethereum_types"] [dependencies] libssz = { version = "0.3.0", path = "../ssz", default-features = false } sha3 = { version = "0.10", default-features = false, optional = true } +# Raw Keccak-f[1600] for the scalar pair path; `asm` uses ARMv8 SHA3 instructions +# through inline assembly (no C toolchain). +keccak = { version = "0.1.6", features = ["asm"], optional = true } +# Runtime CPU feature detection that also works without `std`. +cpufeatures = { version = "0.2", optional = true } ethereum-types = { version = "0.15", default-features = false, optional = true } [build-dependencies] diff --git a/crates/ssz-merkle/src/keccak256.rs b/crates/ssz-merkle/src/keccak256.rs new file mode 100644 index 0000000..1c5137f --- /dev/null +++ b/crates/ssz-merkle/src/keccak256.rs @@ -0,0 +1,654 @@ +//! Default Keccak-256 backend. +//! +//! A Merkle pair is a 64-byte message, which fits in a single Keccak-256 block +//! (the rate is 136 bytes), so each pair costs exactly one Keccak-f\[1600\] +//! permutation. The permutation is run directly on a state with the padding +//! already applied, and a batch of pairs runs several permutations at once in +//! SIMD lanes when the CPU supports it. + +use crate::{MerkleHasher, Node}; + +/// Default Keccak-256 hasher. +/// +/// This is the original Keccak-256 used by Ethereum's execution layer (`0x01` +/// padding), not NIST SHA3-256. +/// +/// Merkle pairs are hashed with one Keccak-f\[1600\] permutation each, skipping +/// the general-purpose sponge. A batch of pairs runs several permutations at +/// once, choosing at runtime: AVX-512 (8 pairs), AVX2 (4) or the ARMv8 SHA3 +/// extension (2). Leftover pairs and other CPUs use the scalar permutation from +/// the `keccak` crate. +pub struct KeccakHasher; + +impl MerkleHasher for KeccakHasher { + fn hash(&self, data: &[u8]) -> [u8; 32] { + use sha3::{Digest, Keccak256}; + Keccak256::digest(data).into() + } + + fn hash_pairs(&self, pairs: &[[Node; 2]], out: &mut [Node]) { + assert_eq!(pairs.len(), out.len(), "one output node per pair"); + + #[cfg(target_arch = "x86_64")] + { + if x86::has_avx512() { + // SAFETY: the CPU supports AVX-512F. + return unsafe { x86::hash_pairs_x8(pairs, out) }; + } + if x86::has_avx2() { + // SAFETY: the CPU supports AVX2. + return unsafe { x86::hash_pairs_x4(pairs, out) }; + } + } + + #[cfg(target_arch = "aarch64")] + if aarch64::has_sha3() { + // SAFETY: the CPU supports the SHA3 extension. + return unsafe { aarch64::hash_pairs_x2(pairs, out) }; + } + + hash_pairs_scalar(pairs, out); + } +} + +/// Keccak padding of a 64-byte message, as state lanes: the `0x01` domain byte +/// goes at byte 64 (lane 8) and the final `0x80` at byte 135, the last byte of +/// the 136-byte rate (lane 16). +const PAD_LANE_8: u64 = 0x01; +const PAD_LANE_16: u64 = 0x80 << 56; + +/// Keccak-f\[1600\] round constants (ι step). +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +const RC: [u64; 24] = [ + 0x0000000000000001, + 0x0000000000008082, + 0x800000000000808a, + 0x8000000080008000, + 0x000000000000808b, + 0x0000000080000001, + 0x8000000080008081, + 0x8000000000008009, + 0x000000000000008a, + 0x0000000000000088, + 0x0000000080008009, + 0x000000008000000a, + 0x000000008000808b, + 0x800000000000008b, + 0x8000000000008089, + 0x8000000000008003, + 0x8000000000008002, + 0x8000000000000080, + 0x000000000000800a, + 0x800000008000000a, + 0x8000000080008081, + 0x8000000000008080, + 0x0000000080000001, + 0x8000000080008008, +]; + +fn hash_pairs_scalar(pairs: &[[Node; 2]], out: &mut [Node]) { + for (pair, out) in pairs.iter().zip(out) { + *out = hash_pair(pair); + } +} + +/// Keccak-256 of `pair[0] || pair[1]`: one permutation of a pre-padded state. +#[inline] +fn hash_pair(pair: &[Node; 2]) -> Node { + let mut state = [0u64; 25]; + let (words, _) = pair.as_flattened().as_chunks::<8>(); + for (lane, word) in state.iter_mut().zip(words) { + *lane = u64::from_le_bytes(*word); + } + state[8] = PAD_LANE_8; + state[16] = PAD_LANE_16; + + keccak::f1600(&mut state); + + let mut digest = [0u8; 32]; + let (bytes, _) = digest.as_chunks_mut::<8>(); + for (bytes, lane) in bytes.iter_mut().zip(&state) { + *bytes = lane.to_le_bytes(); + } + digest +} + +/// Message words of up to `W` pairs, transposed so `words[i][k]` is word `i` +/// of pair `k`. Unused columns stay zero. +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +#[inline(always)] +fn transpose_in(pairs: &[[Node; 2]]) -> [[u64; W]; 8] { + let mut words = [[0u64; W]; 8]; + for (k, pair) in pairs.iter().enumerate() { + let (chunks, _) = pair.as_flattened().as_chunks::<8>(); + for (row, chunk) in words.iter_mut().zip(chunks) { + row[k] = u64::from_le_bytes(*chunk); + } + } + words +} + +/// Write digests from the first four state lanes, where `lanes[i][k]` is lane +/// `i` of the state for pair `k`. +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +#[inline(always)] +fn transpose_out(lanes: &[[u64; W]; 4], out: &mut [Node]) { + for (k, node) in out.iter_mut().enumerate() { + let (chunks, _) = node.as_chunks_mut::<8>(); + for (chunk, row) in chunks.iter_mut().zip(lanes) { + *chunk = row[k].to_le_bytes(); + } + } +} + +/// Keccak-f\[1600\] on `$a: [V; 25]`, where each vector `V` holds the same lane +/// of several independent states. The vector operations come from +/// per-architecture macros, so every intrinsic call is expanded inside the +/// caller's `#[target_feature]` function and can be inlined: +/// +/// - `xor5!(a, b, c, d, e)`: `a ^ b ^ c ^ d ^ e` +/// - `rol1_xor!(a, b)`: `a ^ b.rotate_left(1)` +/// - `xor_rotl!(a, d, N)`: `(a ^ d).rotate_left(N)` for a literal `N` in `0..64` +/// - `chi!(a, b, c)`: `a ^ (!b & c)` +/// - `xor_rc!(a, rc)`: `a ^ rc` in every lane +/// +/// The tests check every backend against `sha3::Keccak256`. +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +macro_rules! keccak_f1600 { + ($a:ident, $xor5:ident, $rol1_xor:ident, $xor_rotl:ident, $chi:ident, $xor_rc:ident) => { + for &rc in &RC { + // θ: column parities, then the correction for each column. + let c0 = $xor5!($a[0], $a[5], $a[10], $a[15], $a[20]); + let c1 = $xor5!($a[1], $a[6], $a[11], $a[16], $a[21]); + let c2 = $xor5!($a[2], $a[7], $a[12], $a[17], $a[22]); + let c3 = $xor5!($a[3], $a[8], $a[13], $a[18], $a[23]); + let c4 = $xor5!($a[4], $a[9], $a[14], $a[19], $a[24]); + let d0 = $rol1_xor!(c4, c1); + let d1 = $rol1_xor!(c0, c2); + let d2 = $rol1_xor!(c1, c3); + let d3 = $rol1_xor!(c2, c4); + let d4 = $rol1_xor!(c3, c0); + + // θ applied, then ρ (rotation) and π (lane permutation): lane + // (X, Y) takes the old lane ((X + 3Y) mod 5, X). + let b0 = $xor_rotl!($a[0], d0, 0); + let b1 = $xor_rotl!($a[6], d1, 44); + let b2 = $xor_rotl!($a[12], d2, 43); + let b3 = $xor_rotl!($a[18], d3, 21); + let b4 = $xor_rotl!($a[24], d4, 14); + let b5 = $xor_rotl!($a[3], d3, 28); + let b6 = $xor_rotl!($a[9], d4, 20); + let b7 = $xor_rotl!($a[10], d0, 3); + let b8 = $xor_rotl!($a[16], d1, 45); + let b9 = $xor_rotl!($a[22], d2, 61); + let b10 = $xor_rotl!($a[1], d1, 1); + let b11 = $xor_rotl!($a[7], d2, 6); + let b12 = $xor_rotl!($a[13], d3, 25); + let b13 = $xor_rotl!($a[19], d4, 8); + let b14 = $xor_rotl!($a[20], d0, 18); + let b15 = $xor_rotl!($a[4], d4, 27); + let b16 = $xor_rotl!($a[5], d0, 36); + let b17 = $xor_rotl!($a[11], d1, 10); + let b18 = $xor_rotl!($a[17], d2, 15); + let b19 = $xor_rotl!($a[23], d3, 56); + let b20 = $xor_rotl!($a[2], d2, 62); + let b21 = $xor_rotl!($a[8], d3, 55); + let b22 = $xor_rotl!($a[14], d4, 39); + let b23 = $xor_rotl!($a[15], d0, 41); + let b24 = $xor_rotl!($a[21], d1, 2); + + // χ along each row. + $a[0] = $chi!(b0, b1, b2); + $a[1] = $chi!(b1, b2, b3); + $a[2] = $chi!(b2, b3, b4); + $a[3] = $chi!(b3, b4, b0); + $a[4] = $chi!(b4, b0, b1); + $a[5] = $chi!(b5, b6, b7); + $a[6] = $chi!(b6, b7, b8); + $a[7] = $chi!(b7, b8, b9); + $a[8] = $chi!(b8, b9, b5); + $a[9] = $chi!(b9, b5, b6); + $a[10] = $chi!(b10, b11, b12); + $a[11] = $chi!(b11, b12, b13); + $a[12] = $chi!(b12, b13, b14); + $a[13] = $chi!(b13, b14, b10); + $a[14] = $chi!(b14, b10, b11); + $a[15] = $chi!(b15, b16, b17); + $a[16] = $chi!(b16, b17, b18); + $a[17] = $chi!(b17, b18, b19); + $a[18] = $chi!(b18, b19, b15); + $a[19] = $chi!(b19, b15, b16); + $a[20] = $chi!(b20, b21, b22); + $a[21] = $chi!(b21, b22, b23); + $a[22] = $chi!(b22, b23, b24); + $a[23] = $chi!(b23, b24, b20); + $a[24] = $chi!(b24, b20, b21); + + // ι. + $a[0] = $xor_rc!($a[0], rc); + } + }; +} + +#[cfg(target_arch = "x86_64")] +mod x86 { + use super::{ + hash_pairs_scalar, transpose_in, transpose_out, Node, PAD_LANE_16, PAD_LANE_8, RC, + }; + use core::arch::x86_64::*; + + // `cpufeatures::new!` expands to the deprecated `u8::max_value()`. + #[allow(deprecated)] + mod cpuid { + cpufeatures::new!(avx512, "avx512f"); + cpufeatures::new!(avx2, "avx2"); + + pub(crate) fn has_avx512() -> bool { + avx512::get() + } + + pub(crate) fn has_avx2() -> bool { + avx2::get() + } + } + pub(super) use cpuid::{has_avx2, has_avx512}; + + /// Smallest leftover batch worth an 8-way permutation instead of scalar ones. + const X8_MIN_PARTIAL: usize = 2; + /// Smallest leftover batch worth a 4-way permutation instead of scalar ones. + const X4_MIN_PARTIAL: usize = 2; + + // AVX-512: `vpternlogq` folds three-input logic into one instruction and + // `vprolq` rotates 64-bit lanes. + macro_rules! avx512_xor5 { + ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr) => { + _mm512_ternarylogic_epi64::<0x96>(_mm512_ternarylogic_epi64::<0x96>($a, $b, $c), $d, $e) + }; + } + macro_rules! avx512_rol1_xor { + ($a:expr, $b:expr) => { + _mm512_xor_si512($a, _mm512_rol_epi64::<1>($b)) + }; + } + macro_rules! avx512_xor_rotl { + ($a:expr, $d:expr, 0) => { + _mm512_xor_si512($a, $d) + }; + ($a:expr, $d:expr, $n:literal) => { + _mm512_rol_epi64::<$n>(_mm512_xor_si512($a, $d)) + }; + } + macro_rules! avx512_chi { + // 0xD2 is the truth table of `a ^ (!b & c)`. + ($a:expr, $b:expr, $c:expr) => { + _mm512_ternarylogic_epi64::<0xD2>($a, $b, $c) + }; + } + macro_rules! avx512_xor_rc { + ($a:expr, $rc:expr) => { + _mm512_xor_si512($a, _mm512_set1_epi64($rc as i64)) + }; + } + + /// Hash up to 8 pairs with one 8-way permutation. + /// + /// # Safety + /// + /// The CPU must support AVX-512F. + #[target_feature(enable = "avx512f")] + unsafe fn hash_batch_x8(pairs: &[[Node; 2]], out: &mut [Node]) { + debug_assert!(pairs.len() <= 8 && pairs.len() == out.len()); + let words = transpose_in::<8>(pairs); + + let mut a = [_mm512_setzero_si512(); 25]; + a[0] = _mm512_loadu_si512(words[0].as_ptr().cast()); + a[1] = _mm512_loadu_si512(words[1].as_ptr().cast()); + a[2] = _mm512_loadu_si512(words[2].as_ptr().cast()); + a[3] = _mm512_loadu_si512(words[3].as_ptr().cast()); + a[4] = _mm512_loadu_si512(words[4].as_ptr().cast()); + a[5] = _mm512_loadu_si512(words[5].as_ptr().cast()); + a[6] = _mm512_loadu_si512(words[6].as_ptr().cast()); + a[7] = _mm512_loadu_si512(words[7].as_ptr().cast()); + a[8] = _mm512_set1_epi64(PAD_LANE_8 as i64); + a[16] = _mm512_set1_epi64(PAD_LANE_16 as i64); + + keccak_f1600!( + a, + avx512_xor5, + avx512_rol1_xor, + avx512_xor_rotl, + avx512_chi, + avx512_xor_rc + ); + + let mut lanes = [[0u64; 8]; 4]; + _mm512_storeu_si512(lanes[0].as_mut_ptr().cast(), a[0]); + _mm512_storeu_si512(lanes[1].as_mut_ptr().cast(), a[1]); + _mm512_storeu_si512(lanes[2].as_mut_ptr().cast(), a[2]); + _mm512_storeu_si512(lanes[3].as_mut_ptr().cast(), a[3]); + transpose_out::<8>(&lanes, out); + } + + /// Hash `pairs` 8 at a time. + /// + /// # Safety + /// + /// The CPU must support AVX-512F. + #[target_feature(enable = "avx512f")] + pub(super) unsafe fn hash_pairs_x8(pairs: &[[Node; 2]], out: &mut [Node]) { + let (batches, rest) = pairs.as_chunks::<8>(); + let (out_batches, out_rest) = out.as_chunks_mut::<8>(); + for (batch, out) in batches.iter().zip(out_batches) { + hash_batch_x8(batch, out); + } + if rest.len() >= X8_MIN_PARTIAL { + hash_batch_x8(rest, out_rest); + } else { + hash_pairs_scalar(rest, out_rest); + } + } + + // AVX2 has no 64-bit rotate or three-input logic, so rotations are two + // shifts and an OR. + macro_rules! avx2_rotl { + ($x:expr, $n:literal) => {{ + let x = $x; + _mm256_or_si256( + _mm256_slli_epi64::<$n>(x), + _mm256_srli_epi64::<{ 64 - $n }>(x), + ) + }}; + } + macro_rules! avx2_xor5 { + ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr) => { + _mm256_xor_si256( + _mm256_xor_si256(_mm256_xor_si256($a, $b), _mm256_xor_si256($c, $d)), + $e, + ) + }; + } + macro_rules! avx2_rol1_xor { + ($a:expr, $b:expr) => { + _mm256_xor_si256($a, avx2_rotl!($b, 1)) + }; + } + macro_rules! avx2_xor_rotl { + ($a:expr, $d:expr, 0) => { + _mm256_xor_si256($a, $d) + }; + ($a:expr, $d:expr, $n:literal) => { + avx2_rotl!(_mm256_xor_si256($a, $d), $n) + }; + } + macro_rules! avx2_chi { + // `_mm256_andnot_si256(b, c)` is `!b & c`. + ($a:expr, $b:expr, $c:expr) => { + _mm256_xor_si256($a, _mm256_andnot_si256($b, $c)) + }; + } + macro_rules! avx2_xor_rc { + ($a:expr, $rc:expr) => { + _mm256_xor_si256($a, _mm256_set1_epi64x($rc as i64)) + }; + } + + /// Hash up to 4 pairs with one 4-way permutation. + /// + /// # Safety + /// + /// The CPU must support AVX2. + #[target_feature(enable = "avx2")] + unsafe fn hash_batch_x4(pairs: &[[Node; 2]], out: &mut [Node]) { + debug_assert!(pairs.len() <= 4 && pairs.len() == out.len()); + let words = transpose_in::<4>(pairs); + + let mut a = [_mm256_setzero_si256(); 25]; + a[0] = _mm256_loadu_si256(words[0].as_ptr().cast()); + a[1] = _mm256_loadu_si256(words[1].as_ptr().cast()); + a[2] = _mm256_loadu_si256(words[2].as_ptr().cast()); + a[3] = _mm256_loadu_si256(words[3].as_ptr().cast()); + a[4] = _mm256_loadu_si256(words[4].as_ptr().cast()); + a[5] = _mm256_loadu_si256(words[5].as_ptr().cast()); + a[6] = _mm256_loadu_si256(words[6].as_ptr().cast()); + a[7] = _mm256_loadu_si256(words[7].as_ptr().cast()); + a[8] = _mm256_set1_epi64x(PAD_LANE_8 as i64); + a[16] = _mm256_set1_epi64x(PAD_LANE_16 as i64); + + keccak_f1600!( + a, + avx2_xor5, + avx2_rol1_xor, + avx2_xor_rotl, + avx2_chi, + avx2_xor_rc + ); + + let mut lanes = [[0u64; 4]; 4]; + _mm256_storeu_si256(lanes[0].as_mut_ptr().cast(), a[0]); + _mm256_storeu_si256(lanes[1].as_mut_ptr().cast(), a[1]); + _mm256_storeu_si256(lanes[2].as_mut_ptr().cast(), a[2]); + _mm256_storeu_si256(lanes[3].as_mut_ptr().cast(), a[3]); + transpose_out::<4>(&lanes, out); + } + + /// Hash `pairs` 4 at a time. + /// + /// # Safety + /// + /// The CPU must support AVX2. + #[target_feature(enable = "avx2")] + pub(super) unsafe fn hash_pairs_x4(pairs: &[[Node; 2]], out: &mut [Node]) { + let (batches, rest) = pairs.as_chunks::<4>(); + let (out_batches, out_rest) = out.as_chunks_mut::<4>(); + for (batch, out) in batches.iter().zip(out_batches) { + hash_batch_x4(batch, out); + } + if rest.len() >= X4_MIN_PARTIAL { + hash_batch_x4(rest, out_rest); + } else { + hash_pairs_scalar(rest, out_rest); + } + } +} + +#[cfg(target_arch = "aarch64")] +mod aarch64 { + use super::{ + hash_pairs_scalar, transpose_in, transpose_out, Node, PAD_LANE_16, PAD_LANE_8, RC, + }; + use core::arch::aarch64::*; + + // `cpufeatures::new!` expands to the deprecated `u8::max_value()`. + #[allow(deprecated)] + mod cpuid { + cpufeatures::new!(sha3_ext, "sha3"); + + pub(crate) fn has_sha3() -> bool { + sha3_ext::get() + } + } + pub(super) use cpuid::has_sha3; + + // The ARMv8.2 SHA3 extension provides three-way XOR (`eor3`), XOR with a + // one-bit rotate (`rax1`), XOR then rotate right (`xar`) and bit-clear-XOR + // (`bcax`), on two 64-bit lanes per register. + macro_rules! sha3_xor5 { + ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr) => { + veor3q_u64(veor3q_u64($a, $b, $c), $d, $e) + }; + } + macro_rules! sha3_rol1_xor { + ($a:expr, $b:expr) => { + vrax1q_u64($a, $b) + }; + } + macro_rules! sha3_xor_rotl { + ($a:expr, $d:expr, 0) => { + veorq_u64($a, $d) + }; + ($a:expr, $d:expr, $n:literal) => { + vxarq_u64::<{ 64 - $n }>($a, $d) + }; + } + macro_rules! sha3_chi { + // `vbcaxq_u64(a, c, b)` is `a ^ (c & !b)`. + ($a:expr, $b:expr, $c:expr) => { + vbcaxq_u64($a, $c, $b) + }; + } + macro_rules! sha3_xor_rc { + ($a:expr, $rc:expr) => { + veorq_u64($a, vdupq_n_u64($rc)) + }; + } + + /// Hash exactly 2 pairs with one 2-way permutation. + /// + /// # Safety + /// + /// The CPU must support the SHA3 extension. + #[target_feature(enable = "neon,sha3")] + unsafe fn hash_batch_x2(pairs: &[[Node; 2]; 2], out: &mut [Node; 2]) { + let words = transpose_in::<2>(pairs); + + let mut a = [vdupq_n_u64(0); 25]; + a[0] = vld1q_u64(words[0].as_ptr()); + a[1] = vld1q_u64(words[1].as_ptr()); + a[2] = vld1q_u64(words[2].as_ptr()); + a[3] = vld1q_u64(words[3].as_ptr()); + a[4] = vld1q_u64(words[4].as_ptr()); + a[5] = vld1q_u64(words[5].as_ptr()); + a[6] = vld1q_u64(words[6].as_ptr()); + a[7] = vld1q_u64(words[7].as_ptr()); + a[8] = vdupq_n_u64(PAD_LANE_8); + a[16] = vdupq_n_u64(PAD_LANE_16); + + keccak_f1600!( + a, + sha3_xor5, + sha3_rol1_xor, + sha3_xor_rotl, + sha3_chi, + sha3_xor_rc + ); + + let mut lanes = [[0u64; 2]; 4]; + vst1q_u64(lanes[0].as_mut_ptr(), a[0]); + vst1q_u64(lanes[1].as_mut_ptr(), a[1]); + vst1q_u64(lanes[2].as_mut_ptr(), a[2]); + vst1q_u64(lanes[3].as_mut_ptr(), a[3]); + transpose_out::<2>(&lanes, out); + } + + /// Hash `pairs` 2 at a time; an odd last pair uses the scalar permutation. + /// + /// # Safety + /// + /// The CPU must support the SHA3 extension. + #[target_feature(enable = "neon,sha3")] + pub(super) unsafe fn hash_pairs_x2(pairs: &[[Node; 2]], out: &mut [Node]) { + let (batches, rest) = pairs.as_chunks::<2>(); + let (out_batches, out_rest) = out.as_chunks_mut::<2>(); + for (batch, out) in batches.iter().zip(out_batches) { + hash_batch_x2(batch, out); + } + hash_pairs_scalar(rest, out_rest); + } +} + +#[cfg(all(test, feature = "alloc"))] +mod tests { + use super::*; + use alloc::vec; + use alloc::vec::Vec; + use sha3::{Digest, Keccak256}; + + /// Pairs whose bytes differ in every position and set high bits, so a + /// misplaced lane or byte-order slip changes the digest. + fn sample_pairs(count: usize) -> Vec<[Node; 2]> { + (0..count) + .map(|i| { + let mut pair = [[0u8; 32]; 2]; + for (j, byte) in pair.as_flattened_mut().iter_mut().enumerate() { + *byte = (i * 131 + j * 29 + 0x80) as u8; + } + pair + }) + .collect() + } + + fn digest(pair: &[Node; 2]) -> Node { + Keccak256::digest(pair.as_flattened()).into() + } + + /// Check a pair-hashing function against `sha3::Keccak256` for every batch + /// count up to five 8-way batches plus remainder, and on all-zero pairs. + fn check_backend(name: &str, hash_pairs: impl Fn(&[[Node; 2]], &mut [Node])) { + for count in 0..=41 { + let pairs = sample_pairs(count); + let mut out = vec![[0u8; 32]; count]; + hash_pairs(&pairs, &mut out); + for (i, (pair, node)) in pairs.iter().zip(&out).enumerate() { + assert_eq!(*node, digest(pair), "{name}: pair {i} of {count}"); + } + } + let zeros = [[[0u8; 32]; 2]; 9]; + let mut out = [[0xffu8; 32]; 9]; + hash_pairs(&zeros, &mut out); + assert!( + out.iter().all(|node| *node == crate::ZERO_HASHES[1]), + "{name}: all-zero pairs must hash to ZERO_HASHES[1]" + ); + } + + #[test] + fn scalar_matches_digest() { + check_backend("scalar", hash_pairs_scalar); + } + + #[test] + fn dispatch_matches_digest() { + check_backend("dispatch", |pairs, out| KeccakHasher.hash_pairs(pairs, out)); + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn avx512_matches_digest() { + if x86::has_avx512() { + // SAFETY: AVX-512F support was just checked. + check_backend("avx512", |pairs, out| unsafe { + x86::hash_pairs_x8(pairs, out) + }); + } + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn avx2_matches_digest() { + if x86::has_avx2() { + // SAFETY: AVX2 support was just checked. + check_backend("avx2", |pairs, out| unsafe { + x86::hash_pairs_x4(pairs, out) + }); + } + } + + #[cfg(target_arch = "aarch64")] + #[test] + fn sha3_extension_matches_digest() { + if aarch64::has_sha3() { + // SAFETY: SHA3 extension support was just checked. + check_backend("sha3", |pairs, out| unsafe { + aarch64::hash_pairs_x2(pairs, out) + }); + } + } + + #[test] + #[should_panic(expected = "one output node per pair")] + fn hash_pairs_rejects_mismatched_output() { + let pairs = sample_pairs(3); + let mut out = [[0u8; 32]; 2]; + KeccakHasher.hash_pairs(&pairs, &mut out); + } +} diff --git a/crates/ssz-merkle/src/lib.rs b/crates/ssz-merkle/src/lib.rs index b97facb..bef0b2b 100644 --- a/crates/ssz-merkle/src/lib.rs +++ b/crates/ssz-merkle/src/lib.rs @@ -16,6 +16,11 @@ pub type Node = [u8; 32]; include!(concat!(env!("OUT_DIR"), "/zero_hashes.rs")); +#[cfg(feature = "keccak-backend")] +mod keccak256; +#[cfg(feature = "keccak-backend")] +pub use keccak256::KeccakHasher; + /// Trait for pluggable Keccak-256 implementations. /// /// Instance method (`&self`) enables passing `&impl MerkleHasher` through the @@ -24,31 +29,33 @@ include!(concat!(env!("OUT_DIR"), "/zero_hashes.rs")); /// Implementations must compute Keccak-256: [`ZERO_HASHES`] is precomputed with /// it, and [`merkleize`] substitutes those entries for all-zero subtrees. pub trait MerkleHasher { + /// Keccak-256 of arbitrary bytes. fn hash(&self, data: &[u8]) -> [u8; 32]; -} - -/// Default Keccak-256 hasher using the `sha3` crate. -/// -/// This is the original Keccak-256 used by Ethereum's execution layer (`0x01` -/// padding), not NIST SHA3-256. -#[cfg(feature = "keccak-backend")] -pub struct KeccakHasher; -#[cfg(feature = "keccak-backend")] -impl MerkleHasher for KeccakHasher { - fn hash(&self, data: &[u8]) -> [u8; 32] { - use sha3::{Digest, Keccak256}; - Keccak256::digest(data).into() + /// Hash each pair, `Keccak256(pair[0] || pair[1])`, into the matching slot of `out`. + /// + /// This is the hot path of Merkleization: [`merkleize`] hashes a whole tree + /// level per call and [`hash_nodes`] passes a single pair, so an + /// implementation can batch pairs or specialize for 64-byte messages. The + /// default hashes one pair at a time through [`hash`](Self::hash). + /// + /// # Panics + /// + /// Panics if `pairs.len() != out.len()`. + fn hash_pairs(&self, pairs: &[[Node; 2]], out: &mut [Node]) { + assert_eq!(pairs.len(), out.len(), "one output node per pair"); + for (pair, out) in pairs.iter().zip(out) { + *out = self.hash(pair.as_flattened()); + } } } /// Hash two nodes together: Keccak256(a || b). #[inline] pub fn hash_nodes(hasher: &impl MerkleHasher, a: &Node, b: &Node) -> Node { - let mut buf = [0u8; 64]; - buf[..32].copy_from_slice(a); - buf[32..].copy_from_slice(b); - hasher.hash(&buf) + let mut out = [[0u8; 32]]; + hasher.hash_pairs(&[[*a, *b]], &mut out); + out[0] } /// Pack serialized bytes into 32-byte chunks, zero-padding the last chunk if needed. @@ -85,7 +92,9 @@ pub fn pack_bits(bits: &[u8], num_bits: usize) -> Vec { /// all zeros, so its root is read from [`ZERO_HASHES`] instead of being hashed. /// /// Cost is `O(chunks.len())` plus one hash per virtual level above the real subtree. -/// Two buffers are allocated and reused (≈ `chunks.len()` and `chunks.len().div_ceil(2)` nodes). +/// Each tree level is hashed in one [`MerkleHasher::hash_pairs`] batch. Two buffers are +/// allocated and reused (≈ `chunks.len() / 2` and `chunks.len() / 4` nodes); the leaves are +/// never copied. #[cfg(feature = "alloc")] pub fn merkleize(hasher: &impl MerkleHasher, chunks: &[Node], limit: Option) -> Node { let count = match limit { @@ -116,36 +125,46 @@ pub fn merkleize(hasher: &impl MerkleHasher, chunks: &[Node], limit: Option = chunks.to_vec(); - let mut next: Vec = Vec::with_capacity(layer.len().div_ceil(2)); - - for zero_hash in ZERO_HASHES.iter().take(real_depth) { - next.clear(); - let (pairs, remainder) = layer.as_chunks::<2>(); - for pair in pairs { - next.push(hash_nodes(hasher, &pair[0], &pair[1])); + // Fold the real chunks one level at a time. Level 0 reads `chunks` in place; + // later levels swap between two buffers allocated once, so no level allocates. + let mut root = if real_depth == 0 { + chunks[0] + } else { + let mut layer: Vec = Vec::with_capacity(chunks.len().div_ceil(2)); + hash_level(hasher, chunks, &ZERO_HASHES[0], &mut layer); + let mut next: Vec = Vec::with_capacity(layer.len().div_ceil(2)); + for zero_hash in &ZERO_HASHES[1..real_depth] { + hash_level(hasher, &layer, zero_hash, &mut next); + core::mem::swap(&mut layer, &mut next); } - if let [odd] = remainder { - next.push(hash_nodes(hasher, odd, zero_hash)); - } - core::mem::swap(&mut layer, &mut next); - } + layer[0] + }; - // `layer[0]` now roots the real-data subtree. Merge in the virtual levels. - let mut root = layer[0]; - for zero_hash in ZERO_HASHES.iter().take(depth).skip(real_depth) { + // `root` now roots the real-data subtree. Merge in the virtual levels. + for zero_hash in &ZERO_HASHES[real_depth..depth] { root = hash_nodes(hasher, &root, zero_hash); } root } +/// Hash one tree level of `layer` into `next`. +/// +/// Full pairs of real nodes are hashed in a single [`MerkleHasher::hash_pairs`] +/// batch. A level with an odd node count ends on the frontier, where the missing +/// right sibling roots an all-zero subtree, so `zero_hash` (the precomputed root +/// of that subtree) stands in for it rather than being hashed. +#[cfg(feature = "alloc")] +fn hash_level(hasher: &impl MerkleHasher, layer: &[Node], zero_hash: &Node, next: &mut Vec) { + let (pairs, remainder) = layer.as_chunks::<2>(); + next.clear(); + next.resize(pairs.len(), [0u8; 32]); + hasher.hash_pairs(pairs, next); + if let [odd] = remainder { + next.push(hash_nodes(hasher, odd, zero_hash)); + } +} + /// Progressive merkleization: a chain of progressively larger binary subtrees. /// /// Creates a 0-terminated sequence of binary subtrees with increasing leaf count: diff --git a/docs/technical-design.md b/docs/technical-design.md index f8f4309..55fd4c6 100644 --- a/docs/technical-design.md +++ b/docs/technical-design.md @@ -196,6 +196,8 @@ merkleize(chunks, limit): **Virtual padding:** rather than allocating `limit` chunks, the implementation uses precomputed `ZERO_HASHES[level]` for any subtree that would be all zeros. This keeps memory usage proportional to the actual number of non-zero chunks. +**Batched levels:** every full pair of a level is handed to `MerkleHasher::hash_pairs` in one call, and `hash_nodes` passes a single pair through the same method. `KeccakHasher` hashes a batch several pairs at a time with SIMD, choosing at runtime: AVX-512 (8 pairs per permutation), AVX2 (4), or ARMv8 SHA3 (2). Pairs left over, and CPUs without those extensions, use one scalar Keccak-f[1600] permutation per pair with the padding applied directly, skipping the general-purpose sponge. + ### Zero hash table `ZERO_HASHES` is a static array of 65 entries (depth 0 through 64), computed at build time by `ssz-merkle/build.rs`: From bcb6a8616459b4082284e79b4b232b1d1c4e4f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:35:08 -0300 Subject: [PATCH 4/5] perf: build AVX2 Keccak rows in registers, use AVX-512 for single pairs AVX2: assemble each state row from 64-bit message words in registers instead of transposing through a stack buffer, whose 64-bit stores read back as one vector stalled on store forwarding. On Zen 5 a full 4-way batch drops from about 378 to 307 ns (94 to 77 ns per hash). AVX-512: one 8-way permutation (about 143 ns) already beats one scalar permutation (about 192 ns), so a leftover single pair uses it too. --- crates/ssz-merkle/src/keccak256.rs | 33 +++++++++++++++++++----------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/crates/ssz-merkle/src/keccak256.rs b/crates/ssz-merkle/src/keccak256.rs index 1c5137f..ebf7948 100644 --- a/crates/ssz-merkle/src/keccak256.rs +++ b/crates/ssz-merkle/src/keccak256.rs @@ -253,9 +253,12 @@ mod x86 { } pub(super) use cpuid::{has_avx2, has_avx512}; - /// Smallest leftover batch worth an 8-way permutation instead of scalar ones. - const X8_MIN_PARTIAL: usize = 2; - /// Smallest leftover batch worth a 4-way permutation instead of scalar ones. + /// Smallest leftover batch worth an 8-way permutation instead of scalar + /// ones. On Zen 5 one 8-way permutation (about 145 ns) already beats one + /// scalar permutation (about 190 ns). + const X8_MIN_PARTIAL: usize = 1; + /// Smallest leftover batch worth a 4-way permutation instead of scalar + /// ones. On Zen 5 a 4-way permutation takes about 250 ns. const X4_MIN_PARTIAL: usize = 2; // AVX-512: `vpternlogq` folds three-input logic into one instruction and @@ -394,23 +397,29 @@ mod x86 { /// Hash up to 4 pairs with one 4-way permutation. /// + /// Each state row is assembled in registers from 64-bit message words. + /// Transposing through a stack buffer instead (64-bit stores read back as + /// one vector) stalls on store forwarding: about 25% slower per full batch + /// on Zen 5. + /// /// # Safety /// /// The CPU must support AVX2. #[target_feature(enable = "avx2")] unsafe fn hash_batch_x4(pairs: &[[Node; 2]], out: &mut [Node]) { debug_assert!(pairs.len() <= 4 && pairs.len() == out.len()); - let words = transpose_in::<4>(pairs); + // Message word `i` of pair `k`, or zero past the end. + let word = |k: usize, i: usize| -> i64 { + match pairs.get(k) { + Some(pair) => i64::from_le_bytes(pair.as_flattened().as_chunks::<8>().0[i]), + None => 0, + } + }; let mut a = [_mm256_setzero_si256(); 25]; - a[0] = _mm256_loadu_si256(words[0].as_ptr().cast()); - a[1] = _mm256_loadu_si256(words[1].as_ptr().cast()); - a[2] = _mm256_loadu_si256(words[2].as_ptr().cast()); - a[3] = _mm256_loadu_si256(words[3].as_ptr().cast()); - a[4] = _mm256_loadu_si256(words[4].as_ptr().cast()); - a[5] = _mm256_loadu_si256(words[5].as_ptr().cast()); - a[6] = _mm256_loadu_si256(words[6].as_ptr().cast()); - a[7] = _mm256_loadu_si256(words[7].as_ptr().cast()); + for (i, row) in a[..8].iter_mut().enumerate() { + *row = _mm256_set_epi64x(word(3, i), word(2, i), word(1, i), word(0, i)); + } a[8] = _mm256_set1_epi64x(PAD_LANE_8 as i64); a[16] = _mm256_set1_epi64x(PAD_LANE_16 as i64); From 135e403207647fae5d2f842b3d7b176a4620c3a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:00:25 -0300 Subject: [PATCH 5/5] docs: correct Keccak leftover-pair and AVX2 notes With AVX-512 a single leftover pair now runs in SIMD, so the docs no longer say leftovers always use the scalar permutation. The AVX2 row assembly measured faster on Zen 5, but the cause is not established: the previous commit attributed it to store-forwarding stalls, while timings also shifted with unrelated code layout changes. --- crates/ssz-merkle/src/keccak256.rs | 11 ++++++----- docs/technical-design.md | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/ssz-merkle/src/keccak256.rs b/crates/ssz-merkle/src/keccak256.rs index ebf7948..0ffa32e 100644 --- a/crates/ssz-merkle/src/keccak256.rs +++ b/crates/ssz-merkle/src/keccak256.rs @@ -16,8 +16,9 @@ use crate::{MerkleHasher, Node}; /// Merkle pairs are hashed with one Keccak-f\[1600\] permutation each, skipping /// the general-purpose sponge. A batch of pairs runs several permutations at /// once, choosing at runtime: AVX-512 (8 pairs), AVX2 (4) or the ARMv8 SHA3 -/// extension (2). Leftover pairs and other CPUs use the scalar permutation from -/// the `keccak` crate. +/// extension (2). With AVX-512 a part-filled batch runs in SIMD too; with AVX2 +/// or SHA3 a single leftover pair, like every pair on other CPUs, uses the +/// scalar permutation from the `keccak` crate. pub struct KeccakHasher; impl MerkleHasher for KeccakHasher { @@ -398,9 +399,9 @@ mod x86 { /// Hash up to 4 pairs with one 4-way permutation. /// /// Each state row is assembled in registers from 64-bit message words. - /// Transposing through a stack buffer instead (64-bit stores read back as - /// one vector) stalls on store forwarding: about 25% slower per full batch - /// on Zen 5. + /// On Zen 5 this measured about 20% faster per full batch than + /// transposing through a stack buffer. The cause is not established: + /// timings of this function shifted with unrelated code layout changes. /// /// # Safety /// diff --git a/docs/technical-design.md b/docs/technical-design.md index 55fd4c6..5eddca0 100644 --- a/docs/technical-design.md +++ b/docs/technical-design.md @@ -196,7 +196,7 @@ merkleize(chunks, limit): **Virtual padding:** rather than allocating `limit` chunks, the implementation uses precomputed `ZERO_HASHES[level]` for any subtree that would be all zeros. This keeps memory usage proportional to the actual number of non-zero chunks. -**Batched levels:** every full pair of a level is handed to `MerkleHasher::hash_pairs` in one call, and `hash_nodes` passes a single pair through the same method. `KeccakHasher` hashes a batch several pairs at a time with SIMD, choosing at runtime: AVX-512 (8 pairs per permutation), AVX2 (4), or ARMv8 SHA3 (2). Pairs left over, and CPUs without those extensions, use one scalar Keccak-f[1600] permutation per pair with the padding applied directly, skipping the general-purpose sponge. +**Batched levels:** every full pair of a level is handed to `MerkleHasher::hash_pairs` in one call, and `hash_nodes` passes a single pair through the same method. `KeccakHasher` hashes a batch several pairs at a time with SIMD, choosing at runtime: AVX-512 (8 pairs per permutation), AVX2 (4), or ARMv8 SHA3 (2). With AVX-512 a part-filled batch runs in SIMD too; with AVX2 or SHA3 a single leftover pair, like every pair on CPUs without those extensions, uses one scalar Keccak-f[1600] permutation with the padding applied directly, skipping the general-purpose sponge. ### Zero hash table