From 6fa83c248bff3448cc417d5d1708109df92060a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:33:21 -0300 Subject: [PATCH 1/5] feat(era): add replay-era sub-command Era files carry every block of 8192 slots plus the end state, which makes them a source of real mainnet, Sepolia and Hoodi history that no node has to still hold. replay-era applies the blocks with the beacon state transition, checking every signature and block state_root, and compares the end of the era with the state stored in the file. Consecutive eras continue from the same in-memory state so a fork upgrade at the boundary runs through our own process_slots. The e2store reader lives in its own file and reads records through the trailing slot indices with seeks, since a recent mainnet era is ~800 MB. Eras at or past a fork this build cannot process (Sepolia gloas) are refused up front rather than failing inside SSZ decoding. Nimbus marks an empty slot in the block index by pointing at byte 0, not with the offset 0 the e2store spec describes; both are read as empty. --- Cargo.lock | 1 + bin/ethlambda/Cargo.toml | 2 + bin/ethlambda/src/command.rs | 5 + bin/ethlambda/src/era/download.rs | 170 ++++++++++++ bin/ethlambda/src/era/e2store.rs | 361 ++++++++++++++++++++++++++ bin/ethlambda/src/era/mod.rs | 246 ++++++++++++++++++ bin/ethlambda/src/era/replay.rs | 355 +++++++++++++++++++++++++ bin/ethlambda/src/era/report.rs | 67 +++++ bin/ethlambda/src/main.rs | 13 +- bin/ethlambda/src/network/built_in.rs | 5 + 10 files changed, 1222 insertions(+), 3 deletions(-) create mode 100644 bin/ethlambda/src/era/download.rs create mode 100644 bin/ethlambda/src/era/e2store.rs create mode 100644 bin/ethlambda/src/era/mod.rs create mode 100644 bin/ethlambda/src/era/replay.rs create mode 100644 bin/ethlambda/src/era/report.rs diff --git a/Cargo.lock b/Cargo.lock index fcf2acfcd..2854d3e80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1962,6 +1962,7 @@ dependencies = [ "serde_json", "serde_yaml_ng", "sha2", + "snap", "tempfile", "thiserror 2.0.18", "tikv-jemallocator", diff --git a/bin/ethlambda/Cargo.toml b/bin/ethlambda/Cargo.toml index 1e239cae3..5f43862b0 100644 --- a/bin/ethlambda/Cargo.toml +++ b/bin/ethlambda/Cargo.toml @@ -47,6 +47,8 @@ serde_yaml_ng.workspace = true hex.workspace = true clap.workspace = true +# Framed snappy, the compression of every record in an `.era` file. +snap = "1.1" reqwest.workspace = true ethlambda-engine.workspace = true thiserror.workspace = true diff --git a/bin/ethlambda/src/command.rs b/bin/ethlambda/src/command.rs index 4bb38c89a..9967ec1f9 100644 --- a/bin/ethlambda/src/command.rs +++ b/bin/ethlambda/src/command.rs @@ -15,6 +15,7 @@ use clap::Parser; use crate::benchmark::BenchmarkOptions; use crate::cli::{BeaconOptions, NodeOptions}; +use crate::era::ReplayEraOptions; use crate::keygen::KeygenOptions; use crate::version; @@ -27,6 +28,7 @@ const EXPLICIT: &[&str] = &[ NODE, BEACON, BENCHMARK, + REPLAY_ERA, VALIDATOR, KEYGEN, "help", @@ -39,6 +41,7 @@ const EXPLICIT: &[&str] = &[ const NODE: &str = "node"; const BEACON: &str = "beacon"; const BENCHMARK: &str = "benchmark"; +const REPLAY_ERA: &str = "replay-era"; const VALIDATOR: &str = "validator"; const KEYGEN: &str = "keygen"; @@ -77,6 +80,8 @@ pub(crate) enum Command { Beacon(BeaconOptions), /// Benchmark block building offline against a controlled workload. Benchmark(BenchmarkOptions), + /// Replay real chain history from `.era` files and check it against the stored states. + ReplayEra(ReplayEraOptions), /// Run the beacon-chain validator client against a beacon node's HTTP API. Validator(crate::validator::ValidatorOptions), /// Generate validator XMSS keys for a genesis. diff --git a/bin/ethlambda/src/era/download.rs b/bin/ethlambda/src/era/download.rs new file mode 100644 index 000000000..ef9b4152f --- /dev/null +++ b/bin/ethlambda/src/era/download.rs @@ -0,0 +1,170 @@ +//! Finding era files on disk and fetching the missing ones. +//! +//! Files come from `https://.era.nimbus.team/`, an autoindex page +//! whose links are the file names. A file is named +//! `--<8 hex>.era`; the trailing root is not predictable +//! from the era number, so the index is the only way to learn it. + +use std::collections::BTreeMap; +use std::io::Write as _; +use std::path::{Path, PathBuf}; + +use eyre::{WrapErr as _, ensure}; +use reqwest::Client; +use tracing::info; + +/// Log a download's progress this often, in bytes. +const PROGRESS_EVERY: u64 = 200 * 1024 * 1024; + +pub(super) fn host(network: &str) -> String { + format!("https://{network}.era.nimbus.team/") +} + +/// Parse `--<8 hex>.era` into its era number. +fn parse_era_name(name: &str, network: &str) -> Option { + let rest = name.strip_prefix(network)?.strip_prefix('-')?; + let rest = rest.strip_suffix(".era")?; + let (era, root) = rest.split_once('-')?; + let well_formed = era.len() == 5 + && era.bytes().all(|b| b.is_ascii_digit()) + && root.len() == 8 + && root.bytes().all(|b| b.is_ascii_hexdigit()); + well_formed.then(|| era.parse().expect("five digits")) +} + +/// The era files an index page lists, by era number. +/// +/// A name appears in the page twice (the link and its text), so the map also +/// dedups. +pub(super) fn parse_listing(page: &str, network: &str) -> BTreeMap { + let prefix = format!("{network}-"); + let mut found = BTreeMap::new(); + for (start, _) in page.match_indices(&prefix) { + // `-NNNNN-xxxxxxxx.era` + let len = prefix.len() + 5 + 1 + 8 + ".era".len(); + let Some(candidate) = page.get(start..start + len) else { + continue; + }; + if let Some(era) = parse_era_name(candidate, network) { + found.insert(era, candidate.to_string()); + } + } + found +} + +pub(super) async fn list_remote( + client: &Client, + network: &str, +) -> eyre::Result> { + let url = host(network); + let page = client + .get(&url) + .send() + .await + .and_then(|response| response.error_for_status()) + .wrap_err_with(|| format!("listing {url}"))? + .text() + .await + .wrap_err_with(|| format!("reading the listing at {url}"))?; + let listing = parse_listing(&page, network); + ensure!(!listing.is_empty(), "{url} lists no {network} era files"); + Ok(listing) +} + +/// The file for `era` already in `dir`, if any. +pub(super) fn find_local(dir: &Path, network: &str, era: u64) -> eyre::Result> { + let entries = std::fs::read_dir(dir).wrap_err_with(|| format!("reading {}", dir.display()))?; + for entry in entries { + let entry = entry?; + if let Some(name) = entry.file_name().to_str() + && parse_era_name(name, network) == Some(era) + { + return Ok(Some(entry.path())); + } + } + Ok(None) +} + +/// Download `file_name` into `dir`, through a `.part` file renamed on success +/// so an interrupted run never leaves something that looks complete. +pub(super) async fn fetch( + client: &Client, + network: &str, + file_name: &str, + dir: &Path, +) -> eyre::Result { + let url = format!("{}{file_name}", host(network)); + let destination = dir.join(file_name); + let partial = dir.join(format!("{file_name}.part")); + + let mut response = client + .get(&url) + .send() + .await + .and_then(|response| response.error_for_status()) + .wrap_err_with(|| format!("downloading {url}"))?; + let total = response.content_length(); + info!(%url, total_bytes = total, "Downloading era file"); + + let mut file = std::fs::File::create(&partial) + .wrap_err_with(|| format!("creating {}", partial.display()))?; + let (mut written, mut next_report) = (0u64, PROGRESS_EVERY); + while let Some(chunk) = response + .chunk() + .await + .wrap_err_with(|| format!("downloading {url}"))? + { + file.write_all(&chunk)?; + written += chunk.len() as u64; + if written >= next_report { + info!(file_name, downloaded_bytes = written, "Download progress"); + next_report += PROGRESS_EVERY; + } + } + file.flush()?; + if let Some(total) = total { + ensure!( + written == total, + "{url} ended after {written} of {total} bytes" + ); + } + drop(file); + std::fs::rename(&partial, &destination) + .wrap_err_with(|| format!("renaming {} into place", partial.display()))?; + Ok(destination) +} + +#[cfg(test)] +mod tests { + use super::*; + + const PAGE: &str = r#"
+mainnet-00289-0a1b2c3d.era     20-Oct-2026 10:00  812345678
+mainnet-00290-deadbeef.era     20-Oct-2026 10:00  712345678
+junk
+sepolia-00001-0a1b2c3d.era
+
"#; + + #[test] + fn the_listing_yields_one_file_per_era_of_this_network() { + let listing = parse_listing(PAGE, "mainnet"); + assert_eq!(listing.len(), 2); + assert_eq!(listing[&290], "mainnet-00290-deadbeef.era"); + assert!(!listing.contains_key(&1)); + } + + #[test] + fn local_files_are_matched_by_network_and_era() { + let dir = tempfile::tempdir().unwrap(); + for name in [ + "mainnet-00290-deadbeef.era", + "mainnet-00291-deadbeef.era.part", + "hoodi-00290-deadbeef.era", + ] { + std::fs::write(dir.path().join(name), b"").unwrap(); + } + let found = find_local(dir.path(), "mainnet", 290).unwrap().unwrap(); + assert!(found.ends_with("mainnet-00290-deadbeef.era")); + assert!(find_local(dir.path(), "mainnet", 291).unwrap().is_none()); + } +} diff --git a/bin/ethlambda/src/era/e2store.rs b/bin/ethlambda/src/era/e2store.rs new file mode 100644 index 000000000..7e0cf4433 --- /dev/null +++ b/bin/ethlambda/src/era/e2store.rs @@ -0,0 +1,361 @@ +//! A reader for the e2store container behind `.era` files. +//! +//! An era file is a sequence of records, each an 8-byte header (`type[2]`, +//! `length` u32 little endian, `reserved` u16 that must be zero) followed by +//! `length` bytes. A file holds one group: +//! +//! ```text +//! Version | block* | era-state | other* | slot-index(block)? | slot-index(state) +//! ``` +//! +//! Blocks and the state are snappy *framed* SSZ. Neither names its fork, so the +//! caller derives it from the slot. The indices sit at the end of the file and +//! are the only way to a record, so the file is read back to front: the last +//! eight bytes of the final index are its `count`, which gives its start, and +//! every offset in an index is measured from the start of that index's own +//! record (negative, since the data precedes it). A slot with no block is +//! marked in one of two ways: the specification says offset zero, but the files +//! nimbus writes point the entry at byte 0 instead, which is the Version record +//! and never a block. Both are read as empty. +//! +//! Records are read one at a time with a seek, never the whole file: a recent +//! mainnet era is around 800 MB. + +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; + +use eyre::{WrapErr as _, ensure, eyre}; + +type Result = eyre::Result; + +/// Record types, as the two bytes that appear on disk. Version (`65 32`) and +/// Empty (`00 00`) records are never looked up: records are reached through the +/// indices, which skips them. +pub(crate) const TYPE_BLOCK: [u8; 2] = [0x01, 0x00]; +pub(crate) const TYPE_STATE: [u8; 2] = [0x02, 0x00]; +pub(crate) const TYPE_SLOT_INDEX: [u8; 2] = [0x69, 0x32]; + +const HEADER_LEN: u64 = 8; + +/// A slot index with more entries than any era could hold is a corrupt tail, +/// not a big file. +const MAX_INDEX_COUNT: u64 = 1 << 20; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Header { + kind: [u8; 2], + length: u32, +} + +/// A `SlotIndex` record: the position of each slot's record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SlotIndex { + pub(crate) start_slot: u64, + offsets: Vec, + /// File position of the index record's own header, which every offset is + /// relative to. + record_start: u64, +} + +impl SlotIndex { + pub(crate) fn count(&self) -> usize { + self.offsets.len() + } + + /// File position of the record for `slot`, or `None` if the slot is empty + /// or outside the index. + pub(crate) fn position(&self, slot: u64) -> Option { + let slot_offset = slot.checked_sub(self.start_slot)?; + let offset = *self.offsets.get(usize::try_from(slot_offset).ok()?)?; + if offset == 0 { + return None; + } + // Nimbus marks an empty slot by pointing at the start of the file. + self.record_start + .checked_add_signed(offset) + .filter(|&position| position != 0) + } + + /// Slots that have a record, in ascending order. + #[cfg(test)] + pub(crate) fn present_slots(&self) -> impl Iterator + '_ { + (0..self.offsets.len() as u64) + .map(|i| self.start_slot + i) + .filter(|&slot| self.position(slot).is_some()) + } +} + +/// An open era file with its indices already read. +#[derive(Debug)] +pub(crate) struct EraFile { + file: File, + block_index: Option, + state_index: SlotIndex, +} + +impl EraFile { + pub(crate) fn open(path: &Path) -> Result { + let mut file = File::open(path).wrap_err_with(|| format!("opening {}", path.display()))?; + let len = file.metadata()?.len(); + Self::from_file(&mut file, len) + .map(|(block_index, state_index)| Self { + file, + block_index, + state_index, + }) + .wrap_err_with(|| format!("reading the indices of {}", path.display())) + } + + fn from_file(file: &mut File, len: u64) -> Result<(Option, SlotIndex)> { + let state_index = read_index_ending_at(file, len)?; + // An era-0 file has no block index; then the bytes before the state + // index are the end of a state record, which will not pass as an index. + let block_index = read_index_ending_at(file, state_index.record_start).ok(); + Ok((block_index, state_index)) + } + + /// The block index: `None` for era 0, which holds a state only. + pub(crate) fn block_index(&self) -> Option<&SlotIndex> { + self.block_index.as_ref() + } + + /// The era's state, decompressed SSZ. + pub(crate) fn state_bytes(&mut self) -> Result> { + let slot = self.state_index.start_slot; + let position = self + .state_index + .position(slot) + .ok_or_else(|| eyre!("the state index has no entry for slot {slot}"))?; + self.read_compressed(position, TYPE_STATE) + } + + /// The block at `slot`, decompressed SSZ, or `None` for an empty slot. + pub(crate) fn block_bytes(&mut self, slot: u64) -> Result>> { + let Some(position) = self.block_index.as_ref().and_then(|i| i.position(slot)) else { + return Ok(None); + }; + self.read_compressed(position, TYPE_BLOCK).map(Some) + } + + fn read_compressed(&mut self, position: u64, expected: [u8; 2]) -> Result> { + let header = read_header(&mut self.file, position)?; + ensure!( + header.kind == expected, + "the record at byte {position} has type {:02x?}, expected {expected:02x?}", + header.kind + ); + // Positioned just past the header by `read_header`. + let payload = (&mut self.file).take(u64::from(header.length)); + let mut decoder = snap::read::FrameDecoder::new(payload); + let mut out = Vec::new(); + decoder + .read_to_end(&mut out) + .wrap_err_with(|| format!("decompressing the record at byte {position}"))?; + Ok(out) + } +} + +fn read_header(file: &mut File, position: u64) -> Result
{ + file.seek(SeekFrom::Start(position))?; + let mut raw = [0u8; HEADER_LEN as usize]; + file.read_exact(&mut raw) + .wrap_err_with(|| format!("reading the record header at byte {position}"))?; + let reserved = u16::from_le_bytes([raw[6], raw[7]]); + ensure!( + reserved == 0, + "the record at byte {position} has a nonzero reserved field" + ); + Ok(Header { + kind: [raw[0], raw[1]], + length: u32::from_le_bytes([raw[2], raw[3], raw[4], raw[5]]), + }) +} + +/// Read the slot-index record whose last byte is just before `end`. +fn read_index_ending_at(file: &mut File, end: u64) -> Result { + ensure!(end >= 32, "no room for a slot index before byte {end}"); + file.seek(SeekFrom::Start(end - 8))?; + let mut raw = [0u8; 8]; + file.read_exact(&mut raw)?; + let count = i64::from_le_bytes(raw); + let count = u64::try_from(count) + .ok() + .filter(|&c| c <= MAX_INDEX_COUNT) + .ok_or_else(|| eyre!("implausible slot index count {count} before byte {end}"))?; + + // header (8) + start slot (8) + count offsets + count (8) + let record_len = count * 8 + 24; + let record_start = end + .checked_sub(record_len) + .ok_or_else(|| eyre!("a slot index of {count} entries does not fit before byte {end}"))?; + let header = read_header(file, record_start)?; + ensure!( + header.kind == TYPE_SLOT_INDEX && u64::from(header.length) == record_len - HEADER_LEN, + "no slot index of {count} entries at byte {record_start}" + ); + + let mut data = vec![0u8; (record_len - HEADER_LEN) as usize]; + file.read_exact(&mut data)?; + let start_slot = u64::from_le_bytes(data[..8].try_into().expect("8 bytes")); + let offsets = data[8..8 + count as usize * 8] + .chunks_exact(8) + .map(|c| i64::from_le_bytes(c.try_into().expect("8 bytes"))) + .collect(); + Ok(SlotIndex { + start_slot, + offsets, + record_start, + }) +} + +#[cfg(test)] +pub(crate) mod tests { + use std::io::Write; + + use super::*; + + const TYPE_VERSION: [u8; 2] = [0x65, 0x32]; + + pub(crate) fn record(kind: [u8; 2], data: &[u8]) -> Vec { + let mut out = kind.to_vec(); + out.extend_from_slice(&(data.len() as u32).to_le_bytes()); + out.extend_from_slice(&[0, 0]); + out.extend_from_slice(data); + out + } + + pub(crate) fn framed(payload: &[u8]) -> Vec { + let mut encoder = snap::write::FrameEncoder::new(Vec::new()); + encoder.write_all(payload).unwrap(); + encoder.into_inner().unwrap() + } + + fn index_record(start_slot: u64, offsets: &[i64]) -> Vec { + let mut data = start_slot.to_le_bytes().to_vec(); + for offset in offsets { + data.extend_from_slice(&offset.to_le_bytes()); + } + data.extend_from_slice(&(offsets.len() as i64).to_le_bytes()); + record(TYPE_SLOT_INDEX, &data) + } + + /// Build an era-shaped file: `blocks` is `(slot, ssz)` within + /// `[start_slot, start_slot + count)`; every other slot is empty. + pub(crate) fn synthetic_file( + start_slot: u64, + count: u64, + blocks: &[(u64, Vec)], + state: &[u8], + with_block_index: bool, + ) -> Vec { + let mut file = record(TYPE_VERSION, &[]); + let mut positions = vec![None; count as usize]; + for (slot, ssz) in blocks { + positions[(slot - start_slot) as usize] = Some(file.len() as i64); + file.extend(record(TYPE_BLOCK, &framed(ssz))); + } + let state_pos = file.len() as i64; + file.extend(record(TYPE_STATE, &framed(state))); + // An unrelated record between the state and the indices. + file.extend(record([0x07, 0x07], b"other")); + + if with_block_index { + let index_start = file.len() as i64; + // Even slots are marked empty the way the spec says (offset 0), odd + // ones the way nimbus writes them (a pointer to byte 0). + let offsets: Vec = positions + .iter() + .enumerate() + .map(|(i, p)| match p { + Some(pos) => pos - index_start, + None if i % 2 == 0 => 0, + None => -index_start, + }) + .collect(); + file.extend(index_record(start_slot, &offsets)); + } + let state_index_start = file.len() as i64; + file.extend(index_record( + start_slot + count, + &[state_pos - state_index_start], + )); + file + } + + fn write(bytes: &[u8]) -> (tempfile::TempDir, std::path::PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("synthetic.era"); + std::fs::write(&path, bytes).unwrap(); + (dir, path) + } + + #[test] + fn header_fields_are_little_endian_and_reserved_is_checked() { + let bytes = record(TYPE_BLOCK, b"abc"); + assert_eq!(&bytes[..8], &[0x01, 0x00, 3, 0, 0, 0, 0, 0]); + + let mut bad = bytes.clone(); + bad[6] = 1; + let (_dir, path) = write(&bad); + let mut file = File::open(path).unwrap(); + assert!(read_header(&mut file, 0).is_err()); + } + + #[test] + fn framed_payloads_round_trip() { + let block = vec![7u8; 100_000]; + let bytes = synthetic_file(16, 4, &[(17, block.clone())], b"state-bytes", true); + let (_dir, path) = write(&bytes); + let mut era = EraFile::open(&path).unwrap(); + assert_eq!(era.block_bytes(17).unwrap().unwrap(), block); + assert_eq!(era.state_bytes().unwrap(), b"state-bytes"); + } + + #[test] + fn empty_slots_have_no_record() { + let blocks = vec![(16, b"first".to_vec()), (19, b"last".to_vec())]; + let bytes = synthetic_file(16, 4, &blocks, b"s", true); + let (_dir, path) = write(&bytes); + let mut era = EraFile::open(&path).unwrap(); + + let index = era.block_index().unwrap(); + assert_eq!(index.start_slot, 16); + assert_eq!(index.count(), 4); + assert_eq!(index.present_slots().collect::>(), vec![16, 19]); + + assert_eq!(era.block_bytes(17).unwrap(), None); + assert_eq!(era.block_bytes(18).unwrap(), None); + assert_eq!(era.block_bytes(99).unwrap(), None); + assert_eq!(era.block_bytes(19).unwrap().unwrap(), b"last"); + } + + #[test] + fn the_file_is_read_from_the_back() { + // The state index's start slot is the era's end slot, and offsets are + // negative: the data precedes the index. + let bytes = synthetic_file(8, 2, &[(9, b"b".to_vec())], b"st", true); + let (_dir, path) = write(&bytes); + let era = EraFile::open(&path).unwrap(); + assert_eq!(era.state_index.start_slot, 10); + assert_eq!(era.state_index.count(), 1); + assert!(era.state_index.offsets[0] < 0); + assert!(era.block_index().unwrap().offsets[1] < 0); + } + + #[test] + fn a_file_without_a_block_index_is_an_era_zero_shaped_one() { + let bytes = synthetic_file(0, 8192, &[], b"genesis", false); + let (_dir, path) = write(&bytes); + let mut era = EraFile::open(&path).unwrap(); + assert!(era.block_index().is_none()); + assert_eq!(era.state_bytes().unwrap(), b"genesis"); + } + + #[test] + fn a_truncated_file_is_an_error() { + let bytes = synthetic_file(0, 2, &[], b"s", true); + let (_dir, path) = write(&bytes[..bytes.len() - 5]); + assert!(EraFile::open(&path).is_err()); + } +} diff --git a/bin/ethlambda/src/era/mod.rs b/bin/ethlambda/src/era/mod.rs new file mode 100644 index 000000000..8c345a9ec --- /dev/null +++ b/bin/ethlambda/src/era/mod.rs @@ -0,0 +1,246 @@ +//! Replaying real chain history from `.era` files (`ethlambda replay-era`). +//! +//! Era files carry every block of 8192 slots plus the state at their end, so +//! they are a source of real mainnet, Sepolia and Hoodi history that no node +//! has to still hold. The command applies the blocks with the beacon state +//! transition, checking every signature and `state_root`, and compares the end +//! with the stored state. See docs/era_replay.md. + +mod download; +mod e2store; +mod replay; +mod report; + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use eyre::{WrapErr as _, bail, ensure, eyre}; +use tracing::info; + +use crate::network::{DEFAULT_NETWORK, NetworkSource, NetworkSpec}; +use replay::{Replay, SLOTS_PER_ERA}; +use report::EraReport; + +/// One `--era` value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EraArg { + Number(u64), + /// The newest era the host lists; needs `--download`. + Latest, +} + +fn parse_era(value: &str) -> Result { + if value == "latest" { + return Ok(EraArg::Latest); + } + value + .parse() + .map(EraArg::Number) + .map_err(|_| format!("`{value}` is neither an era number nor `latest`")) +} + +#[derive(Debug, clap::Args)] +pub(crate) struct ReplayEraOptions { + /// Built-in network name or a directory of published network files. + #[arg(long, default_value = DEFAULT_NETWORK)] + network: String, + /// Directory holding (and receiving) the `.era` files. + #[arg(long)] + era_dir: PathBuf, + /// Era to replay; repeat for several. Consecutive numbers continue from + /// the same in-memory state. `latest` is the newest era on the host. + #[arg(long = "era", required = true, value_parser = parse_era)] + eras: Vec, + /// Fetch missing era files from https://.era.nimbus.team/. + #[arg(long)] + download: bool, + /// Stop each era after this many applied blocks (smoke mode). The + /// end-of-era oracle is skipped. + #[arg(long)] + max_blocks: Option, + /// Write a JSON report of every era to this path. + #[arg(long)] + report: Option, +} + +pub(crate) fn run(options: ReplayEraOptions) -> eyre::Result<()> { + let spec = NetworkSpec::parse(&options.network)?; + let source = NetworkSource::resolve(&spec)?; + let config = source.config(); + // The file prefix and the host name are the config's own name. + let name = config.config_name.as_str().to_string(); + ensure!( + !name.is_empty(), + "the network config has no CONFIG_NAME to name era files by" + ); + let gloas_epoch = gloas_fork_epoch(&spec)?; + + std::fs::create_dir_all(&options.era_dir) + .wrap_err_with(|| format!("creating {}", options.era_dir.display()))?; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let client = crate::checkpoint_sync::build_client()?; + + // Resolve `latest`, then refuse anything this build cannot process before + // downloading gigabytes for it. + let remote = if options.download { + Some(runtime.block_on(download::list_remote(&client, &name))?) + } else { + None + }; + let mut eras = Vec::new(); + for arg in &options.eras { + eras.push(match arg { + EraArg::Number(era) => *era, + EraArg::Latest => { + let remote = remote.as_ref().ok_or_else(|| { + eyre!("`--era latest` needs --download to learn what is newest") + })?; + *remote.keys().next_back().expect("a listing is never empty") + } + }); + } + for &era in &eras { + ensure!(era >= 1, "era 0 holds no blocks; replay starts at era 1"); + check_replayable(era, gloas_epoch, &name)?; + } + + // Files needed: each era's own and its predecessor's. + let wanted: BTreeSet = eras.iter().flat_map(|&era| [era - 1, era]).collect(); + let mut paths = std::collections::BTreeMap::new(); + for era in wanted { + let path = match download::find_local(&options.era_dir, &name, era)? { + Some(path) => path, + None => { + let Some(remote) = &remote else { + bail!( + "no {name} era {era} file in {}; pass --download to fetch it", + options.era_dir.display() + ); + }; + let file_name = remote.get(&era).ok_or_else(|| { + eyre!("{} does not list {name} era {era}", download::host(&name)) + })?; + runtime.block_on(download::fetch(&client, &name, file_name, &options.era_dir))? + } + }; + paths.insert(era, path); + } + + let mut replay = Replay::new(&name, config, options.max_blocks); + let mut reports: Vec = Vec::new(); + for &era in &eras { + let report = replay.era(era, &paths[&(era - 1)], &paths[&era]); + let failed = report.error.is_some(); + reports.push(report); + if failed { + break; + } + } + + if let Some(path) = &options.report { + write_report(path, &reports)?; + } + if let Some(report) = reports.iter().find(|r| r.error.is_some()) { + bail!( + "era {} failed: {}", + report.era, + report.error.as_deref().unwrap_or_default() + ); + } + info!(eras = reports.len(), "All eras replayed"); + Ok(()) +} + +fn write_report(path: &Path, reports: &[EraReport]) -> eyre::Result<()> { + let json = serde_json::to_vec_pretty(reports)?; + std::fs::write(path, json).wrap_err_with(|| format!("writing {}", path.display())) +} + +/// Refuse an era whose closing state belongs to a fork this build cannot +/// process. `Config` has no gloas fork, so decoding such a state as fulu would +/// fail with an SSZ error that names neither the fork nor the fix. +fn check_replayable(era: u64, gloas_epoch: Option, name: &str) -> eyre::Result<()> { + let end_epoch = era * SLOTS_PER_ERA / ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + if let Some(gloas) = gloas_epoch + && end_epoch >= gloas + { + bail!( + "{name} era {era} ends at epoch {end_epoch}, at or past the gloas fork (epoch {gloas}), \ + which this build cannot process; the last replayable era is {}", + (gloas * ethlambda_types::beacon::preset::SLOTS_PER_EPOCH).div_ceil(SLOTS_PER_ERA) - 1 + ); + } + Ok(()) +} + +/// The `GLOAS_FORK_EPOCH` a network's `config.yaml` schedules, which +/// [`ethlambda_types::beacon::config::Config`] drops as an unknown key. +fn gloas_fork_epoch(spec: &NetworkSpec) -> eyre::Result> { + let text = match spec { + NetworkSpec::BuiltIn(network) => network.config_yaml().to_string(), + NetworkSpec::Directory(dir) => { + let path = dir.join("config.yaml"); + std::fs::read_to_string(&path) + .wrap_err_with(|| format!("reading {}", path.display()))? + } + }; + Ok(parse_gloas_epoch(&text)) +} + +fn parse_gloas_epoch(config_yaml: &str) -> Option { + config_yaml + .lines() + .find_map(|line| line.trim().strip_prefix("GLOAS_FORK_EPOCH:")) + .and_then(|value| value.split('#').next()?.trim().parse().ok()) + // An unscheduled fork carries the far-future sentinel. + .filter(|&epoch| epoch != u64::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::network::BuiltInNetwork; + + #[test] + fn the_gloas_epoch_is_read_from_the_config_text() { + assert_eq!( + parse_gloas_epoch("FULU_FORK_EPOCH: 5\nGLOAS_FORK_EPOCH: 353024 # Wed\n"), + Some(353_024) + ); + assert_eq!( + parse_gloas_epoch("GLOAS_FORK_EPOCH: 18446744073709551615"), + None + ); + assert_eq!(parse_gloas_epoch("FULU_FORK_EPOCH: 5"), None); + } + + #[test] + fn eras_past_gloas_are_refused() { + // Sepolia: gloas at epoch 353024 = era 1379. + assert!(check_replayable(1378, Some(353_024), "sepolia").is_ok()); + let err = check_replayable(1379, Some(353_024), "sepolia").unwrap_err(); + assert!( + err.to_string().contains("last replayable era is 1378"), + "{err}" + ); + assert!(check_replayable(9999, None, "mainnet").is_ok()); + } + + #[test] + fn era_arguments_parse() { + assert_eq!(parse_era("290"), Ok(EraArg::Number(290))); + assert_eq!(parse_era("latest"), Ok(EraArg::Latest)); + assert!(parse_era("soon").is_err()); + } + + #[test] + fn built_in_networks_name_their_era_files_by_config_name() { + for network in BuiltInNetwork::ALL { + let source = NetworkSource::resolve(&NetworkSpec::BuiltIn(network)).unwrap(); + assert_eq!(source.config().config_name.as_str(), network.name()); + } + } +} diff --git a/bin/ethlambda/src/era/replay.rs b/bin/ethlambda/src/era/replay.rs new file mode 100644 index 000000000..5f4aaeedb --- /dev/null +++ b/bin/ethlambda/src/era/replay.rs @@ -0,0 +1,355 @@ +//! Replaying one era of blocks through the beacon state transition. +//! +//! Era file `N` holds the blocks of slots `[(N-1)*8192, N*8192)` and the state +//! at slot `N*8192`. The state stored in file `N-1` is therefore exactly the +//! state the first block of file `N` builds on. Applying every block with +//! `validate_result` on checks each proposer signature and each `state_root` +//! against what the chain committed to; the state stored in file `N` is an +//! independent end-of-era oracle. +//! +//! # The block at a state's own slot +//! +//! An era state is the state *with* the block at its slot applied, if there is +//! one, and that block is stored in the next file. So a starting state may +//! already contain file `N`'s first block (skipped here: its slot is not above +//! the state's), and the closing state may be one block short of what the next +//! file would add. The oracle is written around that, see [`Replay::oracle`]. + +use std::path::Path; +use std::time::Instant; + +use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; +use ethlambda_state_transition::beacon::stf::{self, ExecutionEngine}; +use ethlambda_types::beacon::config::Config; +use ethlambda_types::beacon::containers::{BeaconState, SignedBeaconBlock}; +use ethlambda_types::beacon::preset; +use ethlambda_types::primitives::H256; +use eyre::{WrapErr as _, ensure, eyre}; +use tracing::info; + +use super::e2store::EraFile; +use super::report::{EraReport, Outcome}; + +/// Blocks between progress lines. +const PROGRESS_EVERY: u64 = 512; + +/// Slots per era file. +pub(super) const SLOTS_PER_ERA: u64 = preset::SLOTS_PER_HISTORICAL_ROOT as u64; + +fn hex(root: H256) -> String { + format!("0x{}", hex::encode(root.0)) +} + +/// State carried from one era into the next, so a fork upgrade at the boundary +/// is crossed by this node's own `process_slots`. +struct Carried { + era: u64, + state: BeaconState, +} + +pub(super) struct Replay<'a> { + network: &'a str, + config: &'a Config, + engine: ExecutionEngine, + /// One for the whole run, as the node holds one across its imports. + committees: CommitteeCache, + max_blocks: Option, + carried: Option, +} + +impl<'a> Replay<'a> { + pub(super) fn new(network: &'a str, config: &'a Config, max_blocks: Option) -> Self { + Self { + network, + config, + engine: ExecutionEngine::valid(), + committees: CommitteeCache::default(), + max_blocks, + carried: None, + } + } + + /// Replay era `era`, starting from the state in `previous` unless the last + /// era replayed was `era - 1`. + /// + /// Returns the report whether or not the era passed. + pub(super) fn era(&mut self, era: u64, previous: &Path, current: &Path) -> EraReport { + let mut report = EraReport::new(self.network, era, self.max_blocks); + let started = Instant::now(); + info!(network = self.network, era, "Era replay starting"); + let result = self.replay(era, previous, current, &mut report); + report.elapsed_secs = started.elapsed().as_secs_f64(); + if let Err(err) = result { + // A failed era leaves a partially advanced state behind. + self.carried = None; + report.error = Some(format!("{err:#}")); + } + info!( + network = self.network, + era, + blocks_applied = report.blocks_applied, + elapsed_secs = format_args!("{:.1}", report.elapsed_secs), + mean_ms_per_block = format_args!("{:.1}", report.mean_ms_per_block), + ok = report.error.is_none(), + "Era replay finished" + ); + report + } + + fn replay( + &mut self, + era: u64, + previous: &Path, + current: &Path, + report: &mut EraReport, + ) -> eyre::Result<()> { + ensure!(era >= 1, "era 0 holds no blocks to replay"); + let first_slot = (era - 1) * SLOTS_PER_ERA; + let end_slot = era * SLOTS_PER_ERA; + + let mut file = EraFile::open(current)?; + let block_index = file + .block_index() + .ok_or_else(|| eyre!("{} has no block index", current.display()))?; + ensure!( + block_index.start_slot == first_slot && block_index.count() as u64 == SLOTS_PER_ERA, + "{} indexes {} slots from {}, expected {SLOTS_PER_ERA} from {first_slot}", + current.display(), + block_index.count(), + block_index.start_slot + ); + + let mut state = match self.carried.take() { + Some(carried) if carried.era + 1 == era => { + info!(era, "Continuing from the previous era's in-memory state"); + carried.state + } + _ => self.load_state(previous, first_slot)?, + }; + + let apply_started = Instant::now(); + let mut block_roots: Vec<(u64, H256)> = Vec::new(); + let mut stopped_early = false; + for slot in first_slot..end_slot { + let Some(bytes) = file.block_bytes(slot)? else { + report.skipped_empty_slots += 1; + continue; + }; + let fork = self.config.fork_at_epoch(slot / preset::SLOTS_PER_EPOCH); + let block = SignedBeaconBlock::from_ssz(fork, &bytes).map_err(|err| { + eyre!("era {era} slot {slot}: block does not decode as {fork:?}: {err:?}") + })?; + ensure!( + block.slot() == slot, + "era {era}: the record indexed at slot {slot} holds a block for slot {}", + block.slot() + ); + drop(bytes); + block_roots.push((slot, block.message_hash_tree_root())); + + if slot <= state.slot() { + report.blocks_already_in_state += 1; + continue; + } + if self + .max_blocks + .is_some_and(|max| report.blocks_applied >= max) + { + stopped_early = true; + break; + } + let expected_state_root = block.state_root(); + stf::state_transition( + &mut state, + &block, + true, + self.config, + &self.engine, + &self.committees, + ) + .map_err(|err| { + eyre!( + "era {era} slot {slot}: block rejected: {err:?} \ + (block commits to state_root {})", + hex(expected_state_root) + ) + })?; + report.blocks_applied += 1; + if report.blocks_applied.is_multiple_of(PROGRESS_EVERY) { + info!( + era, + %slot, + blocks_applied = report.blocks_applied, + elapsed_secs = format_args!("{:.0}", apply_started.elapsed().as_secs_f64()), + "Era replay progress" + ); + } + } + report.apply_secs = apply_started.elapsed().as_secs_f64(); + if report.blocks_applied > 0 { + report.mean_ms_per_block = report.apply_secs * 1000.0 / report.blocks_applied as f64; + } + + // `--max-blocks` may exactly equal the era's block count, which is no + // early stop; only a break leaves the state short of the era's end. + if stopped_early { + for name in ORACLE_CHECKS { + report.check( + name, + Outcome::Skipped, + Some("--max-blocks stopped the era early".into()), + ); + } + return Ok(()); + } + + self.oracle(era, state, &block_roots, &mut file, report) + } + + fn load_state(&self, path: &Path, expected_slot: u64) -> eyre::Result { + let mut file = EraFile::open(path)?; + let bytes = file.state_bytes()?; + let slot = BeaconState::slot_from_ssz(&bytes) + .map_err(|err| eyre!("{}: state has no readable slot: {err:?}", path.display()))?; + ensure!( + slot == expected_slot, + "{} holds a state at slot {slot}, expected {expected_slot}", + path.display() + ); + let fork = self.config.fork_at_epoch(slot / preset::SLOTS_PER_EPOCH); + BeaconState::from_ssz(fork, &bytes).map_err(|err| { + eyre!( + "{}: state does not decode as {fork:?}: {err:?}", + path.display() + ) + }) + } + + /// Compare the replayed chain with the state stored in the era's own file. + /// + /// Three checks, in the order they can be made without file `N+1`: + /// + /// 1. `state_roots[end - 1]` of the stored state is the root of the state + /// after the last slot before the era boundary, which is what + /// `process_slot` records on the way out of it. Compared with our own + /// state advanced to `end - 1`. It holds whether or not a block sits at + /// `end`, because that block is applied after the boundary. + /// 2. Every block's root against the stored `block_roots` window. + /// 3. When the stored state's `latest_block_header` is older than `end`, + /// slot `end` is empty, so advancing our state to `end` must reproduce + /// the stored state outright, including any fork upgrade there. When a + /// block sits at `end` the stored state includes it and we do not have + /// it, so the full comparison is skipped. + fn oracle( + &mut self, + era: u64, + mut state: BeaconState, + block_roots: &[(u64, H256)], + file: &mut EraFile, + report: &mut EraReport, + ) -> eyre::Result<()> { + let end_slot = era * SLOTS_PER_ERA; + let bytes = file.state_bytes()?; + let slot = BeaconState::slot_from_ssz(&bytes) + .map_err(|err| eyre!("era {era}: stored state has no readable slot: {err:?}"))?; + ensure!( + slot == end_slot, + "era {era}: the stored state is at slot {slot}, expected {end_slot}" + ); + let fork = self.config.fork_at_epoch(slot / preset::SLOTS_PER_EPOCH); + let stored = BeaconState::from_ssz(fork, &bytes) + .map_err(|err| eyre!("era {era}: stored state does not decode as {fork:?}: {err:?}"))?; + drop(bytes); + + // 1. state_roots at the last slot before the boundary. + let last_slot = end_slot - 1; + if state.slot() < last_slot { + stf::process_slots(&mut state, last_slot, self.config) + .wrap_err_with(|| format!("era {era}: advancing to slot {last_slot}"))?; + } + ensure!( + state.slot() == last_slot, + "era {era}: state is at slot {} after the era's blocks, expected {last_slot}", + state.slot() + ); + let expected = stored.state_roots()[(last_slot % SLOTS_PER_ERA) as usize]; + let actual = state.hash_tree_root(); + let result = compare( + "state_roots[last slot]", + era, + last_slot, + expected, + actual, + report, + ); + result?; + + // 2. block roots. + for &(slot, root) in block_roots { + let expected = stored.block_roots()[(slot % SLOTS_PER_ERA) as usize]; + compare("block_roots", era, slot, expected, root, report)?; + } + report.check( + "block_roots", + Outcome::Passed, + Some(format!("{} blocks", block_roots.len())), + ); + + // 3. the whole state, when slot `end` is empty. + if stored.latest_block_header().slot < end_slot { + stf::process_slots(&mut state, end_slot, self.config) + .wrap_err_with(|| format!("era {era}: advancing to slot {end_slot}"))?; + compare( + "full_state_root", + era, + end_slot, + stored.hash_tree_root(), + state.hash_tree_root(), + report, + )?; + } else { + report.check( + "full_state_root", + Outcome::Skipped, + Some(format!( + "slot {end_slot} holds a block, stored in the next era file" + )), + ); + } + + self.carried = Some(Carried { era, state }); + Ok(()) + } +} + +const ORACLE_CHECKS: [&str; 3] = ["state_roots[last slot]", "block_roots", "full_state_root"]; + +/// Record one root comparison; a mismatch is a hard failure naming everything +/// needed to look it up. +fn compare( + name: &'static str, + era: u64, + slot: u64, + expected: H256, + actual: H256, + report: &mut EraReport, +) -> eyre::Result<()> { + if expected == actual { + // `block_roots` is recorded once for all its blocks by the caller. + if name != "block_roots" { + report.check( + name, + Outcome::Passed, + Some(format!("slot {slot}: {}", hex(actual))), + ); + } + return Ok(()); + } + let detail = format!( + "era {era} slot {slot}: {name} mismatch: expected {}, actual {}", + hex(expected), + hex(actual) + ); + report.check(name, Outcome::Failed, Some(detail.clone())); + Err(eyre!(detail)) +} diff --git a/bin/ethlambda/src/era/report.rs b/bin/ethlambda/src/era/report.rs new file mode 100644 index 000000000..ac2e4c48f --- /dev/null +++ b/bin/ethlambda/src/era/report.rs @@ -0,0 +1,67 @@ +//! The JSON a `replay-era` run leaves behind for CI to archive. + +use serde::Serialize; + +/// What one end-of-era check concluded. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum Outcome { + Passed, + Failed, + Skipped, +} + +#[derive(Debug, Clone, Serialize)] +pub(super) struct OracleCheck { + pub(super) name: &'static str, + pub(super) outcome: Outcome, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) detail: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub(super) struct EraReport { + pub(super) network: String, + pub(super) era: u64, + pub(super) blocks_applied: u64, + /// Blocks in the file at or below the starting state's slot, which the + /// state already includes. + pub(super) blocks_already_in_state: u64, + pub(super) skipped_empty_slots: u64, + /// Wall time for the whole era: decoding, blocks and the oracle. + pub(super) elapsed_secs: f64, + /// Wall time spent applying blocks (decode plus state transition). + pub(super) apply_secs: f64, + pub(super) mean_ms_per_block: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) max_blocks: Option, + pub(super) oracle: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) error: Option, +} + +impl EraReport { + pub(super) fn new(network: &str, era: u64, max_blocks: Option) -> Self { + Self { + network: network.to_string(), + era, + blocks_applied: 0, + blocks_already_in_state: 0, + skipped_empty_slots: 0, + elapsed_secs: 0.0, + apply_secs: 0.0, + mean_ms_per_block: 0.0, + max_blocks, + oracle: Vec::new(), + error: None, + } + } + + pub(super) fn check(&mut self, name: &'static str, outcome: Outcome, detail: Option) { + self.oracle.push(OracleCheck { + name, + outcome, + detail, + }); + } +} diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 5d6f07db6..0a16c15d7 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -4,6 +4,7 @@ mod benchmark; mod checkpoint_sync; mod cli; mod command; +mod era; mod fd_limit; mod keygen; mod network; @@ -96,6 +97,12 @@ fn main() -> eyre::Result<()> { init_benchmark_logging()?; benchmark::run(options) } + // Synchronous and CPU-bound like the benchmark; only the downloads + // start a runtime of their own. + Command::ReplayEra(options) => { + init_info_stderr_logging()?; + era::run(options) + } Command::Validator(options) => { init_node_logging()?; validator::run(options) @@ -103,7 +110,7 @@ fn main() -> eyre::Result<()> { // Key generation is synchronous, CPU-bound work whose product is files, // so it runs on this thread too and leaves stdout alone. Command::Keygen(options) => { - init_keygen_logging()?; + init_info_stderr_logging()?; keygen::run(options) } } @@ -134,10 +141,10 @@ fn init_benchmark_logging() -> eyre::Result<()> { .wrap_err("failed to set global tracing subscriber") } -/// Keygen logging: INFO and above, on stderr. Key generation takes minutes for a +/// Keygen and era-replay logging: INFO and above, on stderr. Key generation takes minutes for a /// large validator set, so progress has to be visible; stderr keeps stdout free /// for a machine-readable summary to claim later. -fn init_keygen_logging() -> eyre::Result<()> { +fn init_info_stderr_logging() -> eyre::Result<()> { let filter = EnvFilter::builder() .with_default_directive(tracing::Level::INFO.into()) .from_env_lossy(); diff --git a/bin/ethlambda/src/network/built_in.rs b/bin/ethlambda/src/network/built_in.rs index 1ff084d36..2d7df5c24 100644 --- a/bin/ethlambda/src/network/built_in.rs +++ b/bin/ethlambda/src/network/built_in.rs @@ -47,6 +47,11 @@ impl BuiltInNetwork { Self::ALL.into_iter().find(|network| network.name() == name) } + /// The embedded `config.yaml`, as published. + pub(crate) fn config_yaml(self) -> &'static str { + self.chain().config_yaml + } + fn chain(self) -> &'static EmbeddedChain { match self { Self::Mainnet => &MAINNET, From a1f744fb4e34778e99a41103f7716439a981a39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:33:26 -0300 Subject: [PATCH 2/5] ci: replay real eras on every push to main that touches the STF The spec fixtures are tiny, so only real history shows STF divergences at fork upgrades and at mainnet-sized registries. The workflow builds nothing: it waits for the image docker_publish.yaml pushes for the same commit and runs replay-era in it, one job per fork-boundary era per network (Sepolia never past its last pre-gloas era). A failed push run reports the commit and the failing entries to Slack. --- .github/workflows/era_replay.yml | 188 +++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 .github/workflows/era_replay.yml diff --git a/.github/workflows/era_replay.yml b/.github/workflows/era_replay.yml new file mode 100644 index 000000000..ae61aa04a --- /dev/null +++ b/.github/workflows/era_replay.yml @@ -0,0 +1,188 @@ +name: Era replay + +# Replays real chain history from nimbus .era files through the beacon state +# transition and checks the result against the states stored in the files. +# Nothing is built here: it runs the image docker_publish.yaml pushes for the +# same commit. See docs/era_replay.md. + +on: + push: + branches: [main] + paths: + - "crates/blockchain/state_transition/**" + - "crates/common/types/**" + - "crates/common/ssz-tree/**" + - "bin/ethlambda/src/era/**" + - ".github/workflows/era_replay.yml" + workflow_dispatch: + inputs: + image: + description: "Image to run (default: the sha tag of the dispatched ref)" + required: false + type: string + eras: + description: >- + Override the matrix: space-separated `network:era[+era...]` entries, + e.g. `mainnet:290 hoodi:198+199`. `latest` is accepted as an era. + required: false + type: string + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + wait-for-image: + name: Wait for image + runs-on: ubuntu-latest + timeout-minutes: 80 + outputs: + image: ${{ steps.image.outputs.image }} + steps: + - name: Name the image + id: image + env: + INPUT_IMAGE: ${{ inputs.image }} + SHA: ${{ github.sha }} + run: | + echo "image=${INPUT_IMAGE:-ghcr.io/lambdaclass/ethlambda:sha-${SHA::7}}" >> "$GITHUB_OUTPUT" + + - name: Wait for the image tag to exist + env: + IMAGE: ${{ steps.image.outputs.image }} + run: | + deadline=$((SECONDS + 75 * 60)) + until docker manifest inspect "$IMAGE" > /dev/null 2>&1; do + if (( SECONDS >= deadline )); then + echo "::error::$IMAGE was not published within 75 minutes" + exit 1 + fi + echo "$IMAGE not published yet, retrying in 60 s" + sleep 60 + done + echo "$IMAGE is available" + + matrix: + name: Select eras + runs-on: ubuntu-latest + outputs: + include: ${{ steps.select.outputs.include }} + steps: + - name: Build the matrix + id: select + env: + OVERRIDE: ${{ inputs.eras }} + run: | + default='[ + {"network":"mainnet","eras":"290"}, + {"network":"mainnet","eras":"566"}, + {"network":"mainnet","eras":"758"}, + {"network":"mainnet","eras":"1053"}, + {"network":"mainnet","eras":"1422"}, + {"network":"mainnet","eras":"1607+1608"}, + {"network":"mainnet","eras":"latest"}, + {"network":"hoodi","eras":"8"}, + {"network":"hoodi","eras":"198+199"}, + {"network":"hoodi","eras":"latest"}, + {"network":"sepolia","eras":"222"}, + {"network":"sepolia","eras":"518"}, + {"network":"sepolia","eras":"869"}, + {"network":"sepolia","eras":"1065+1066"}, + {"network":"sepolia","eras":"1378"} + ]' + if [[ -n "$OVERRIDE" ]]; then + include=$(printf '%s\n' $OVERRIDE | jq -R -s -c \ + 'split("\n") | map(select(length > 0) | split(":") | {network: .[0], eras: .[1]})') + else + include=$(jq -c . <<< "$default") + fi + echo "include=$include" >> "$GITHUB_OUTPUT" + + replay: + name: ${{ matrix.network }} era ${{ matrix.eras }} + needs: [wait-for-image, matrix] + runs-on: ubuntu-latest + timeout-minutes: 330 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.matrix.outputs.include) }} + steps: + - name: Replay + env: + IMAGE: ${{ needs.wait-for-image.outputs.image }} + NETWORK: ${{ matrix.network }} + ERAS: ${{ matrix.eras }} + run: | + mkdir -p "$RUNNER_TEMP/eras" + era_args=() + for era in ${ERAS//+/ }; do + era_args+=(--era "$era") + done + docker run --rm -v "$RUNNER_TEMP/eras:/eras" "$IMAGE" replay-era \ + --network "$NETWORK" --era-dir /eras --download \ + --report /eras/report.json "${era_args[@]}" + + - name: Name the report + if: always() + env: + NETWORK: ${{ matrix.network }} + ERAS: ${{ matrix.eras }} + run: | + if [[ -f "$RUNNER_TEMP/eras/report.json" ]]; then + mv "$RUNNER_TEMP/eras/report.json" "$RUNNER_TEMP/report-$NETWORK-${ERAS//+/-}.json" + fi + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: era-replay-${{ matrix.network }}-${{ strategy.job-index }} + path: ${{ runner.temp }}/report-*.json + if-no-files-found: ignore + + notify: + name: Post failure to Slack + needs: [wait-for-image, matrix, replay] + if: failure() && github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Find the failing entries + id: failed + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.run_id }} + REPOSITORY: ${{ github.repository }} + run: | + names=$(gh api --paginate \ + "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs" \ + --jq '.jobs[] | select(.conclusion == "failure") | .name' \ + | paste -sd, - | sed 's/,/, /g') + echo "names=$names" >> "$GITHUB_OUTPUT" + + - name: Build Slack payload + # Context values go through the environment and `jq` assembles the + # JSON, so a name containing `"` or a newline cannot break the payload. + env: + FAILED: ${{ steps.failed.outputs.names }} + SHA: ${{ github.sha }} + COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + # The backticks are literal Slack mrkdwn, not command substitution. + # shellcheck disable=SC2016 + text=$(printf '❌ *Era replay* failed on <%s|`%s`>\nFailing: %s\n<%s|View run>' \ + "$COMMIT_URL" "${SHA::7}" "${FAILED:-see the run}" "$RUN_URL") + jq -n --arg text "$text" \ + '{blocks: [{type: "section", text: {type: "mrkdwn", text: $text}}]}' \ + > era_replay_slack.json + + - name: Post to Slack + env: + SLACK_WEBHOOK: ${{ secrets.ETHLAMBDA_GENERAL_SLACK_WEBHOOK }} + run: bash .github/scripts/publish_slack.sh era_replay_slack.json From 6bacfb397657f6940c6a3d77855221a6b065a738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:33:28 -0300 Subject: [PATCH 3/5] docs: describe the era replay battery Covers what the end-of-era oracle does and does not compare, why a block at the end slot limits the full-state check, the measured runtimes, and the quadratic phase0 epoch processing that makes early mainnet eras impractical. --- CLAUDE.md | 3 + docs/SUMMARY.md | 1 + docs/cli.md | 6 +- docs/era_replay.md | 145 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 docs/era_replay.md diff --git a/CLAUDE.md b/CLAUDE.md index d4cfc073f..146697e91 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -735,6 +735,9 @@ GENESIS_VALIDATORS: - `crates/blockchain/tests/signature_spectests.rs` - `crates/blockchain/state_transition/tests/stf_spectests.rs` (state transition) +### Era Replay +`ethlambda replay-era` replays real mainnet/Sepolia/Hoodi blocks from nimbus `.era` files through the beacon STF and compares the end of each era with the stored state. `.github/workflows/era_replay.yml` runs it on the published image after every push to `main` that touches the STF, types or ssz-tree. See [`docs/era_replay.md`](docs/era_replay.md). + ### Running Tests ```bash cargo test --workspace --profile release-fast # All workspace tests diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 97cf4f901..311798283 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -32,4 +32,5 @@ # Development - [Benchmarking](./benchmarking.md) +- [Era Replay](./era_replay.md) - [Spec Deviations](./spec_deviations.md) diff --git a/docs/cli.md b/docs/cli.md index 4b665e167..c01c0d3ec 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -8,7 +8,9 @@ | `ethlambda beacon ` | The Ethereum Beacon Chain, as a gossip follower | | `ethlambda ` | `node`: the subcommand is injected | -Two subcommands follow no chain. `ethlambda benchmark` is the offline +Three subcommands follow no chain. `ethlambda replay-era` replays real chain +history from `.era` files and checks it against the states stored in them; see +[`era_replay.md`](./era_replay.md) for its flags and checks. `ethlambda benchmark` is the offline benchmarking harness, covering block building (`benchmark synthetic`, run through `make bench`) and block import (`benchmark import fetch` / `benchmark import replay`). See [`benchmarking.md`](./benchmarking.md) for @@ -22,7 +24,7 @@ against a beacon node over the standard REST API, documented in clap has no native default subcommand, so the binary rewrites its own argv before parsing. If the first argument is not `node`, `beacon`, `benchmark`, -`validator`, `help`, `-h`, `--help`, `-V`, or `--version`, then `node` is +`replay-era`, `validator`, `help`, `-h`, `--help`, `-V`, or `--version`, then `node` is inserted ahead of it. The function is `inject_default_subcommand` in `bin/ethlambda/src/command.rs`. Every subcommand has to be listed there: a missing one would have `node` inserted ahead of it and become unreachable. diff --git a/docs/era_replay.md b/docs/era_replay.md new file mode 100644 index 000000000..86dd6013f --- /dev/null +++ b/docs/era_replay.md @@ -0,0 +1,145 @@ +# Era replay + +`ethlambda replay-era` replays real chain history from `.era` files through the +Beacon Chain state transition. Era files carry every block of 8192 slots plus +the state at the end of them, so they are history that no node has to still +hold, unlike `benchmark import`, which reads a running node's Beacon API. The +command exists to catch state-transition divergences on real data across every +fork, including the upgrade itself. + +## What a run checks + +For era `N`, file `N` holds the blocks of slots `[(N-1)*8192, N*8192)` and the +state at slot `N*8192`. The command decodes the state stored in file `N-1`, +then applies each block of file `N` in slot order with +`stf::state_transition(.., validate_result = true, ..)` and one `CommitteeCache` +for the whole run: + +| Check | Where | +|---|---| +| Proposer signature, randao, attestations, sync aggregate, deposits, exits | every block, always on | +| Block `state_root` equals the post-state's root | every block | +| Block root equals `block_roots[slot % 8192]` of the stored end state | after the era | +| `state_roots[N*8192 - 1]` of the stored state equals our state's root at the last slot before the boundary | after the era | +| Our full state root at `N*8192` equals the stored state's | after the era, only when slot `N*8192` is empty | + +The execution payload is not run: the engine answers `valid`, so structural +payload checks (parent hash, `prev_randao`, timestamp, commitment limit) run and +the EL verdict does not. + +### The block at the end slot + +An era state is the state with the block at its own slot applied, and that +block is stored in the next file. Two consequences: a starting state may +already contain the first block of file `N` (blocks at or below the state's slot +are skipped and counted as `blocks_already_in_state`), and the closing state +may include a block we do not have. The `state_roots` check needs nothing from +file `N+1`, which is why it is the primary end-of-era check; the full-state +comparison runs only when the stored state's `latest_block_header` is older than +`N*8192`, i.e. the slot is empty. + +### Chaining and fork upgrades + +Consecutive `--era` values continue from the same in-memory state without +re-decoding. A fork upgrade is exercised by our own `process_slots`: replaying +era 1607 (mainnet) from file 1606's electra state, advancing to slot +`1607*8192` runs the fulu upgrade, and the result is compared with the fulu state +stored in file 1607 when that slot is empty. Era 1608 then applies the first +fulu blocks. A non-consecutive era starts fresh from its predecessor's file. + +Fork eras are `fork_epoch / 256`: + +| Network | altair | bellatrix | capella | deneb | electra | fulu | gloas | +|---|---|---|---|---|---|---|---| +| mainnet | 290 | 566 | 758 | 1053 | 1422 | 1607 | n/a | +| sepolia | in era 0 | in era 0 | 222 | 518 | 869 | 1065 | 1379 | +| hoodi | genesis | genesis | genesis | genesis | 8 | 198 | n/a | + +An era whose closing state is at or past a fork this build cannot process +(Sepolia gloas, era 1379 and later) is refused up front with the last +replayable era named. The check reads `GLOAS_FORK_EPOCH` from the network's +`config.yaml`, which `Config` itself drops. + +## Command line + +``` +ethlambda replay-era --network --era-dir \ + --era [--era ...] [--download] [--max-blocks K] [--report ] +``` + +| Flag | Meaning | +|---|---| +| `--network` | Resolved like `beacon`'s: a built-in name or a directory of published files. Its `CONFIG_NAME` names the era files and the host | +| `--era-dir` | Where `--<8 hex>.era` files are read from and downloaded to | +| `--era` | Repeatable. A number, or `latest` (newest era the host lists; needs `--download`) | +| `--download` | Fetch missing files from `https://.era.nimbus.team/`, matching the `--` prefix in the index page. Written to `.part`, renamed on success | +| `--max-blocks` | Smoke mode: stop each era after K applied blocks. Per-block checks still run; the end-of-era checks are skipped and reported as `skipped` | +| `--report` | JSON array with one entry per era: network, era, `blocks_applied`, `blocks_already_in_state`, `skipped_empty_slots`, `elapsed_secs`, `apply_secs`, `mean_ms_per_block`, the oracle checks with their outcome, and `error` | + +Every era needs two files, its own and its predecessor's: about 1.5 GB for a +recent mainnet era. The command logs one line per era start and end and a +progress line every 512 blocks, and exits non-zero on any failure, after +writing the report. A mismatch names the era, slot, expected and actual roots. + +The e2store reader (`bin/ethlambda/src/era/e2store.rs`) reads the trailing slot +indices first and then seeks to single records, so a file is never held in memory +whole; only the one state being decoded is. + +## CI + +`.github/workflows/era_replay.yml` runs on pushes to `main` that touch the +state transition, the types, the ssz-tree or the era module, and on manual +dispatch. It builds nothing: it waits for `docker_publish.yaml`'s +`ghcr.io/lambdaclass/ethlambda:sha-<7>` tag and runs `replay-era` in it, one job +per entry: + +| Network | Entries | +|---|---| +| mainnet | 290, 566, 758, 1053, 1422, 1607+1608, latest | +| hoodi | 8, 198+199, latest | +| sepolia | 222, 518, 869, 1065+1066, 1378 | + +Each job uploads its `report.json`; a failed push run posts the commit, the +failing entries and the run URL to Slack. Manual dispatch takes an `image` and +an `eras` override such as `mainnet:290 hoodi:198+199`. + +## Measured runtimes + +Measured on an Apple-silicon laptop with the `release-fast` binary, one core +busy (`mean ms/block` covers block decode, the state transition and epoch +processing; downloads and the end-of-era checks are excluded): + +| Run | Result | +|---|---| +| Sepolia era 1, full (7627 blocks, crosses altair and bellatrix) | 116 s, 15.2 ms/block, all three end-of-era checks as below | +| Sepolia eras 2+3+4, chained | about 120 s each, 15 ms/block, `state_roots` and `block_roots` passed; `Continuing from the previous era's in-memory state` logged | +| Sepolia era 31, full (slot 253952 empty) | 108 s, 15.9 ms/block, `state_roots`, `block_roots` and `full_state_root` all passed | +| Mainnet era 1888 (latest, fulu), `--max-blocks 64` | 653 ms/block, every proposer signature and `state_root` passed; 1.5 GB download in about 2 min | +| Hoodi era 8 (electra), `--max-blocks 256` | 4.6 s/block (1178 s; a second replay was running on the machine) | + +A full recent era is 8192 blocks, so mainnet at 0.65 s/block is about 90 minutes +and Hoodi at 4.6 s/block is about 10 hours: the Hoodi entries do not fit the +330 minute job limit of a hosted runner at that speed. + +### Phase0 eras are not practical yet + +The beacon STF's phase0 epoch processing is quadratic in the validator count: +`get_base_reward` calls `get_total_active_balance`, which walks the whole +registry, once per validator per reward component +(`stf/epoch/rewards.rs`). Mainnet era 2 (about 20k validators) spends roughly +30 s per epoch boundary, 512 blocks in 484 s, and mainnet era 290 (about 280k +validators) did not finish its first epoch boundary in 30 minutes. Phase0 eras +past the first few hundred need that total hoisted out of the per-validator +loop first; the spec fixtures are far too small to show it. Eras from altair +on do not have the problem. + +The end-of-era checks, as observed on real files: slot `N*8192` held a block in +every boundary checked except era 31 on Sepolia, so `full_state_root` is usually +skipped and `state_roots[last slot]` plus `block_roots` carry the check. + +### File format notes + +Verified against real nimbus files: the slot index marks an empty slot by +pointing at byte 0 (the Version record) rather than by offset 0 as the +specification says, so both are read as empty. Block and state records are +snappy framed; offsets are relative to the index record's own start. From 2317028fbf6f31752cc9220e3dacb7b154c83449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:56:47 -0300 Subject: [PATCH 4/5] ci(era): drop mainnet era 290, make Hoodi a post-fork smoke run Mainnet era 290 never got through one phase0 epoch boundary in 30+ minutes: get_base_reward recomputes get_total_active_balance per validator, which is quadratic in the registry. It leaves the default matrix (the dispatch override can still request it) until that is fixed. A Hoodi block takes ~4.6 s against ~0.65 s on mainnet, so full eras exceed the job limit. Hoodi now runs 512-block smoke entries. Era file F = fork_epoch/256 holds the last pre-fork blocks, so a smoke run of it never reaches the upgrade. The entries are the files after the forks (9 for electra, 199 for fulu) plus latest, each separate since a smoke run cannot chain. --max-blocks with several --era values silently replayed K blocks of each from a fresh state; it is now a CLI error. The matrix and the `eras` override take a per-entry `@K` suffix (e.g. hoodi:9@512). Docs get a corrected era/fork table and a Known limits section. --- .github/workflows/era_replay.yml | 44 +++++++++++++++++++++++++------- bin/ethlambda/src/era/mod.rs | 28 ++++++++++++++++++-- docs/era_replay.md | 41 +++++++++++++++++++++-------- 3 files changed, 92 insertions(+), 21 deletions(-) diff --git a/.github/workflows/era_replay.yml b/.github/workflows/era_replay.yml index ae61aa04a..739121728 100644 --- a/.github/workflows/era_replay.yml +++ b/.github/workflows/era_replay.yml @@ -22,8 +22,10 @@ on: type: string eras: description: >- - Override the matrix: space-separated `network:era[+era...]` entries, - e.g. `mainnet:290 hoodi:198+199`. `latest` is accepted as an era. + Override the matrix: space-separated `network:era[+era...][@K]` + entries, e.g. `mainnet:290 hoodi:199 hoodi:9@512`. `latest` is + accepted as an era. `@K` makes the entry a smoke run of K blocks + (a single era only, since a smoke run stops mid-era). required: false type: string @@ -76,17 +78,33 @@ jobs: env: OVERRIDE: ${{ inputs.eras }} run: | + # Era file N holds slots [(N-1)*8192, N*8192), so the file of a fork + # epoch E (N = E/256) holds the LAST pre-fork blocks and the upgrade + # runs in our own closing process_slots. The `F+F+1` entries add the + # first post-fork blocks. + # + # Mainnet era 290 (the last phase0 file, closing in altair) is left + # out: phase0 epoch processing is quadratic in the registry + # (get_base_reward recomputes get_total_active_balance per + # validator, stf/epoch/rewards.rs), and ~280k validators did not get + # through one epoch boundary in 30 minutes. Re-add `290` once that is + # fixed; the `eras` dispatch override can still request it. + # + # Hoodi blocks take ~4.6 s each (mainnet ~0.65 s), so Hoodi runs as + # smoke tests of 512 blocks that start right after a fork: era 9 + # holds the first electra blocks (electra epoch 2048 = file 8), era + # 199 the first fulu blocks (fulu epoch 50688 = file 198). They are + # separate entries because a smoke run stops mid-era and cannot chain. default='[ - {"network":"mainnet","eras":"290"}, {"network":"mainnet","eras":"566"}, {"network":"mainnet","eras":"758"}, {"network":"mainnet","eras":"1053"}, {"network":"mainnet","eras":"1422"}, {"network":"mainnet","eras":"1607+1608"}, {"network":"mainnet","eras":"latest"}, - {"network":"hoodi","eras":"8"}, - {"network":"hoodi","eras":"198+199"}, - {"network":"hoodi","eras":"latest"}, + {"network":"hoodi","eras":"9","max_blocks":"512"}, + {"network":"hoodi","eras":"199","max_blocks":"512"}, + {"network":"hoodi","eras":"latest","max_blocks":"512"}, {"network":"sepolia","eras":"222"}, {"network":"sepolia","eras":"518"}, {"network":"sepolia","eras":"869"}, @@ -94,15 +112,19 @@ jobs: {"network":"sepolia","eras":"1378"} ]' if [[ -n "$OVERRIDE" ]]; then - include=$(printf '%s\n' $OVERRIDE | jq -R -s -c \ - 'split("\n") | map(select(length > 0) | split(":") | {network: .[0], eras: .[1]})') + include=$(printf '%s\n' $OVERRIDE | jq -R -s -c ' + split("\n") | map(select(length > 0) + | split(":") as [$network, $rest] + | ($rest | split("@")) as [$eras, $max] + | {network: $network, eras: $eras} + + (if $max then {max_blocks: $max} else {} end))') else include=$(jq -c . <<< "$default") fi echo "include=$include" >> "$GITHUB_OUTPUT" replay: - name: ${{ matrix.network }} era ${{ matrix.eras }} + name: ${{ matrix.network }} era ${{ matrix.eras }}${{ matrix.max_blocks && format(' (first {0} blocks)', matrix.max_blocks) || '' }} needs: [wait-for-image, matrix] runs-on: ubuntu-latest timeout-minutes: 330 @@ -116,12 +138,16 @@ jobs: IMAGE: ${{ needs.wait-for-image.outputs.image }} NETWORK: ${{ matrix.network }} ERAS: ${{ matrix.eras }} + MAX_BLOCKS: ${{ matrix.max_blocks }} run: | mkdir -p "$RUNNER_TEMP/eras" era_args=() for era in ${ERAS//+/ }; do era_args+=(--era "$era") done + if [[ -n "$MAX_BLOCKS" ]]; then + era_args+=(--max-blocks "$MAX_BLOCKS") + fi docker run --rm -v "$RUNNER_TEMP/eras:/eras" "$IMAGE" replay-era \ --network "$NETWORK" --era-dir /eras --download \ --report /eras/report.json "${era_args[@]}" diff --git a/bin/ethlambda/src/era/mod.rs b/bin/ethlambda/src/era/mod.rs index 8c345a9ec..fddeae7b2 100644 --- a/bin/ethlambda/src/era/mod.rs +++ b/bin/ethlambda/src/era/mod.rs @@ -54,8 +54,9 @@ pub(crate) struct ReplayEraOptions { /// Fetch missing era files from https://.era.nimbus.team/. #[arg(long)] download: bool, - /// Stop each era after this many applied blocks (smoke mode). The - /// end-of-era oracle is skipped. + /// Stop the era after this many applied blocks (smoke mode). The + /// end-of-era oracle is skipped. Takes a single `--era`, since a smoke run + /// stops mid-era and has no state to chain from. #[arg(long)] max_blocks: Option, /// Write a JSON report of every era to this path. @@ -63,7 +64,20 @@ pub(crate) struct ReplayEraOptions { report: Option, } +/// `--max-blocks` leaves the state mid-era, so a second `--era` could only +/// start fresh from its predecessor's file and apply K more blocks: a run that +/// looks chained and is not. Refuse it instead. +fn check_max_blocks(eras: &[EraArg], max_blocks: Option) -> eyre::Result<()> { + ensure!( + max_blocks.is_none() || eras.len() <= 1, + "--max-blocks stops an era mid-way, so it cannot be combined with several --era values; \ + run each era as its own smoke run" + ); + Ok(()) +} + pub(crate) fn run(options: ReplayEraOptions) -> eyre::Result<()> { + check_max_blocks(&options.eras, options.max_blocks)?; let spec = NetworkSpec::parse(&options.network)?; let source = NetworkSource::resolve(&spec)?; let config = source.config(); @@ -236,6 +250,16 @@ mod tests { assert!(parse_era("soon").is_err()); } + #[test] + fn max_blocks_refuses_chained_eras() { + let one = [EraArg::Number(9)]; + let two = [EraArg::Number(9), EraArg::Number(10)]; + assert!(check_max_blocks(&one, Some(512)).is_ok()); + assert!(check_max_blocks(&two, None).is_ok()); + let err = check_max_blocks(&two, Some(512)).unwrap_err(); + assert!(err.to_string().contains("--max-blocks"), "{err}"); + } + #[test] fn built_in_networks_name_their_era_files_by_config_name() { for network in BuiltInNetwork::ALL { diff --git a/docs/era_replay.md b/docs/era_replay.md index 86dd6013f..bafaada4d 100644 --- a/docs/era_replay.md +++ b/docs/era_replay.md @@ -47,11 +47,14 @@ era 1607 (mainnet) from file 1606's electra state, advancing to slot stored in file 1607 when that slot is empty. Era 1608 then applies the first fulu blocks. A non-consecutive era starts fresh from its predecessor's file. -Fork eras are `fork_epoch / 256`: +Fork files are `F = fork_epoch / 256`. File `F` holds the **last pre-fork** +blocks (slots up to the fork epoch), and the upgrade happens in our closing +`process_slots`, so replaying `F` exercises the upgrade at the very end. The +first post-fork blocks are in file `F+1`, which is why CI chains `F+F+1`: | Network | altair | bellatrix | capella | deneb | electra | fulu | gloas | |---|---|---|---|---|---|---|---| -| mainnet | 290 | 566 | 758 | 1053 | 1422 | 1607 | n/a | +| mainnet | 290 (not in CI, see Known limits) | 566 | 758 | 1053 | 1422 | 1607 | n/a | | sepolia | in era 0 | in era 0 | 222 | 518 | 869 | 1065 | 1379 | | hoodi | genesis | genesis | genesis | genesis | 8 | 198 | n/a | @@ -73,7 +76,7 @@ ethlambda replay-era --network --era-dir \ | `--era-dir` | Where `--<8 hex>.era` files are read from and downloaded to | | `--era` | Repeatable. A number, or `latest` (newest era the host lists; needs `--download`) | | `--download` | Fetch missing files from `https://.era.nimbus.team/`, matching the `--` prefix in the index page. Written to `.part`, renamed on success | -| `--max-blocks` | Smoke mode: stop each era after K applied blocks. Per-block checks still run; the end-of-era checks are skipped and reported as `skipped` | +| `--max-blocks` | Smoke mode: stop the era after K applied blocks. Per-block checks still run; the end-of-era checks are skipped and reported as `skipped`. Takes a single `--era`; combining it with several is a CLI error, since a smoke run stops mid-era and cannot chain | | `--report` | JSON array with one entry per era: network, era, `blocks_applied`, `blocks_already_in_state`, `skipped_empty_slots`, `elapsed_secs`, `apply_secs`, `mean_ms_per_block`, the oracle checks with their outcome, and `error` | Every era needs two files, its own and its predecessor's: about 1.5 GB for a @@ -95,13 +98,31 @@ per entry: | Network | Entries | |---|---| -| mainnet | 290, 566, 758, 1053, 1422, 1607+1608, latest | -| hoodi | 8, 198+199, latest | -| sepolia | 222, 518, 869, 1065+1066, 1378 | +| mainnet | 566, 758, 1053, 1422, 1607+1608, latest (full eras) | +| sepolia | 222, 518, 869, 1065+1066, 1378 (full eras) | +| hoodi | 9, 199, latest, each `--max-blocks 512` (smoke) | + +Hoodi era 9 is the first electra blocks (electra epoch 2048 is the end of +file 8) and era 199 the first fulu blocks (fulu epoch 50688 is the end of file +198). They are separate entries because a smoke run cannot chain. Each job uploads its `report.json`; a failed push run posts the commit, the failing entries and the run URL to Slack. Manual dispatch takes an `image` and -an `eras` override such as `mainnet:290 hoodi:198+199`. +an `eras` override of space-separated `network:era[+era...][@K]` entries, where +`@K` adds `--max-blocks K` (single era only), e.g. +`mainnet:290 hoodi:199 hoodi:9@512`. + +## Known limits + +- **Phase0 is quadratic.** Mainnet era 290 is excluded from the matrix, see + "Phase0 eras are not practical yet" below. The override can still request it. +- **Hoodi per-block cost is under investigation.** About 4.6 s per block + against 0.65 s on mainnet, so a full Hoodi era would exceed the job limit and + Hoodi runs as 512-block smoke tests. +- **Smoke runs do not cross fork boundaries.** A `--max-blocks` run stops + mid-era, so it never reaches the closing `process_slots` where the upgrade + runs, and it skips the end-of-era checks. That is why the Hoodi entries start + at the file after the fork (9, 199) and exercise the first post-fork blocks. ## Measured runtimes @@ -114,11 +135,11 @@ processing; downloads and the end-of-era checks are excluded): | Sepolia era 1, full (7627 blocks, crosses altair and bellatrix) | 116 s, 15.2 ms/block, all three end-of-era checks as below | | Sepolia eras 2+3+4, chained | about 120 s each, 15 ms/block, `state_roots` and `block_roots` passed; `Continuing from the previous era's in-memory state` logged | | Sepolia era 31, full (slot 253952 empty) | 108 s, 15.9 ms/block, `state_roots`, `block_roots` and `full_state_root` all passed | -| Mainnet era 1888 (latest, fulu), `--max-blocks 64` | 653 ms/block, every proposer signature and `state_root` passed; 1.5 GB download in about 2 min | -| Hoodi era 8 (electra), `--max-blocks 256` | 4.6 s/block (1178 s; a second replay was running on the machine) | +| Mainnet era 1888 (latest, fulu blocks), `--max-blocks 64` | 653 ms/block, every proposer signature and `state_root` passed; 1.5 GB download in about 2 min | +| Hoodi era 8, `--max-blocks 256` | 4.6 s/block (1178 s; a second replay was running on the machine) | A full recent era is 8192 blocks, so mainnet at 0.65 s/block is about 90 minutes -and Hoodi at 4.6 s/block is about 10 hours: the Hoodi entries do not fit the +and Hoodi at 4.6 s/block is about 10 hours: a full Hoodi era does not fit the 330 minute job limit of a hosted runner at that speed. ### Phase0 eras are not practical yet From c043a41585b80b6a2186479676e73f14749c6bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:15:54 -0300 Subject: [PATCH 5/5] ci(era): replay full Hoodi eras Hoodi ran as 512-block smoke tests because a deneb-era block cost about 4.6 s: get_base_reward_per_increment scans the whole registry once per attestation. This assumes #648 (per-epoch total-active-balance cache) is merged, which removes that scan and lets full Hoodi eras fit the job limit. The entries are 8 (last deneb blocks, electra upgrade in the closing process_slots), 198+199 (fulu upgrade plus the first fulu era) and latest. Mainnet 290 stays excluded: #648 does not touch the phase0 path. --- .github/workflows/era_replay.yml | 18 ++++++++++-------- docs/era_replay.md | 23 +++++++++++++---------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/era_replay.yml b/.github/workflows/era_replay.yml index 739121728..3a868d7b5 100644 --- a/.github/workflows/era_replay.yml +++ b/.github/workflows/era_replay.yml @@ -90,11 +90,13 @@ jobs: # through one epoch boundary in 30 minutes. Re-add `290` once that is # fixed; the `eras` dispatch override can still request it. # - # Hoodi blocks take ~4.6 s each (mainnet ~0.65 s), so Hoodi runs as - # smoke tests of 512 blocks that start right after a fork: era 9 - # holds the first electra blocks (electra epoch 2048 = file 8), era - # 199 the first fulu blocks (fulu epoch 50688 = file 198). They are - # separate entries because a smoke run stops mid-era and cannot chain. + # Hoodi: era 8 holds the last deneb blocks (electra epoch 2048) and + # the electra upgrade runs in our closing process_slots; 198+199 is + # the fulu upgrade (fulu epoch 50688 = file 198) plus the first fulu + # era. Full Hoodi eras fit the job limit only with the per-epoch + # total-active-balance cache from #648; without it a deneb-era block + # costs about 4.6 s, because get_base_reward_per_increment scans the + # registry per attestation. default='[ {"network":"mainnet","eras":"566"}, {"network":"mainnet","eras":"758"}, @@ -102,9 +104,9 @@ jobs: {"network":"mainnet","eras":"1422"}, {"network":"mainnet","eras":"1607+1608"}, {"network":"mainnet","eras":"latest"}, - {"network":"hoodi","eras":"9","max_blocks":"512"}, - {"network":"hoodi","eras":"199","max_blocks":"512"}, - {"network":"hoodi","eras":"latest","max_blocks":"512"}, + {"network":"hoodi","eras":"8"}, + {"network":"hoodi","eras":"198+199"}, + {"network":"hoodi","eras":"latest"}, {"network":"sepolia","eras":"222"}, {"network":"sepolia","eras":"518"}, {"network":"sepolia","eras":"869"}, diff --git a/docs/era_replay.md b/docs/era_replay.md index bafaada4d..0ca914247 100644 --- a/docs/era_replay.md +++ b/docs/era_replay.md @@ -100,11 +100,11 @@ per entry: |---|---| | mainnet | 566, 758, 1053, 1422, 1607+1608, latest (full eras) | | sepolia | 222, 518, 869, 1065+1066, 1378 (full eras) | -| hoodi | 9, 199, latest, each `--max-blocks 512` (smoke) | +| hoodi | 8, 198+199, latest (full eras, need the #648 cache, see Known limits) | -Hoodi era 9 is the first electra blocks (electra epoch 2048 is the end of -file 8) and era 199 the first fulu blocks (fulu epoch 50688 is the end of file -198). They are separate entries because a smoke run cannot chain. +Hoodi era 8 is the last deneb blocks, with the electra upgrade (epoch 2048) in +our closing `process_slots`; `198+199` is the fulu upgrade (epoch 50688) plus +the first fulu era. Each job uploads its `report.json`; a failed push run posts the commit, the failing entries and the run URL to Slack. Manual dispatch takes an `image` and @@ -116,13 +116,15 @@ an `eras` override of space-separated `network:era[+era...][@K]` entries, where - **Phase0 is quadratic.** Mainnet era 290 is excluded from the matrix, see "Phase0 eras are not practical yet" below. The override can still request it. -- **Hoodi per-block cost is under investigation.** About 4.6 s per block - against 0.65 s on mainnet, so a full Hoodi era would exceed the job limit and - Hoodi runs as 512-block smoke tests. +- **Full Hoodi eras depend on #648.** Without the per-epoch + total-active-balance cache from #648, a deneb-era block costs about 4.6 s + against 0.65 s on mainnet, because `get_base_reward_per_increment` scans the + whole registry once per attestation (`stf/deneb.rs:359`, + `stf/electra.rs:764`). A full era would then exceed the job limit. - **Smoke runs do not cross fork boundaries.** A `--max-blocks` run stops mid-era, so it never reaches the closing `process_slots` where the upgrade - runs, and it skips the end-of-era checks. That is why the Hoodi entries start - at the file after the fork (9, 199) and exercise the first post-fork blocks. + runs, and it skips the end-of-era checks. The CI matrix has no smoke entries; + they are available through the `eras` override (`@K`). ## Measured runtimes @@ -140,7 +142,8 @@ processing; downloads and the end-of-era checks are excluded): A full recent era is 8192 blocks, so mainnet at 0.65 s/block is about 90 minutes and Hoodi at 4.6 s/block is about 10 hours: a full Hoodi era does not fit the -330 minute job limit of a hosted runner at that speed. +330 minute job limit of a hosted runner at that speed, which is why the matrix +relies on #648. ### Phase0 eras are not practical yet