From 3d563e07b4cd734d45c09d9b9740049201ba60a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:30:03 -0300 Subject: [PATCH 01/20] feat(engine): read shouldOverrideBuilder from getPayloadV5 The Engine API's fulu answer carries the execution client's wish to build locally whatever a builder pays; the type dropped it, so the builder flow could not honour it. --- crates/net/engine/src/building.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/net/engine/src/building.rs b/crates/net/engine/src/building.rs index 6ff109248..e7ac9559f 100644 --- a/crates/net/engine/src/building.rs +++ b/crates/net/engine/src/building.rs @@ -118,6 +118,9 @@ pub struct BuiltPayload { pub blobs_bundle: BlobsBundle, /// The EIP-7685 request list, each entry its type byte then its data. pub execution_requests: Vec>, + /// The engine's `shouldOverrideBuilder`: it wants this payload built + /// locally whatever a builder bid pays. Absent from an answer means false. + pub should_override_builder: bool, } /// `engine_getPayloadV6`'s answer, decoded: [`BuiltPayload`] with Amsterdam's @@ -151,6 +154,8 @@ pub(crate) struct GetPayloadV5Response { block_value: String, blobs_bundle: BlobsBundleJson, #[serde(default)] + should_override_builder: bool, + #[serde(default)] execution_requests: Vec, } @@ -259,6 +264,7 @@ impl TryFrom for BuiltPayload { block_value: parse_uint256(&response.block_value)?, blobs_bundle, execution_requests: decode_requests(&response.execution_requests)?, + should_override_builder: response.should_override_builder, }) } } @@ -454,6 +460,29 @@ mod tests { assert_eq!(built.execution_requests, vec![vec![0x00, 0xaa]]); } + #[test] + fn a_v5_answer_carries_should_override_builder_and_defaults_it_to_false() { + let mut json = response_json(); + json["shouldOverrideBuilder"] = true.into(); + let response: GetPayloadV5Response = serde_json::from_value(json).unwrap(); + assert!( + BuiltPayload::try_from(response) + .unwrap() + .should_override_builder + ); + + let mut json = response_json(); + json.as_object_mut() + .unwrap() + .remove("shouldOverrideBuilder"); + let response: GetPayloadV5Response = serde_json::from_value(json).unwrap(); + assert!( + !BuiltPayload::try_from(response) + .unwrap() + .should_override_builder + ); + } + #[test] fn a_wrong_width_hash_is_a_decode_error() { let mut json = response_json(); From 951ff737cf026f54836b2632f57f10c899a1deaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:31:35 -0300 Subject: [PATCH 02/20] feat(rpc): remember the registrations forwarded to the builder Block production asks the builder only for proposers that registered through this node, and needs their registered gas limit to check a bid. A newer registration replaces an older one, and one not resent for two epochs is gone, since validator clients resend every epoch. --- .../net/rpc/src/beacon/builder_flow/cache.rs | 139 ++++++++++++++++++ crates/net/rpc/src/beacon/builder_flow/mod.rs | 5 + crates/net/rpc/src/beacon/mod.rs | 1 + 3 files changed, 145 insertions(+) create mode 100644 crates/net/rpc/src/beacon/builder_flow/cache.rs create mode 100644 crates/net/rpc/src/beacon/builder_flow/mod.rs diff --git a/crates/net/rpc/src/beacon/builder_flow/cache.rs b/crates/net/rpc/src/beacon/builder_flow/cache.rs new file mode 100644 index 000000000..f0d0903a2 --- /dev/null +++ b/crates/net/rpc/src/beacon/builder_flow/cache.rs @@ -0,0 +1,139 @@ +//! The registrations this node forwarded to its builder, so block +//! production knows which proposers use the builder, and with what gas +//! limit. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use ethlambda_types::beacon::builder::ValidatorRegistrationV1; +use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch}; + +/// How many epochs a registration lasts without being sent again. Validator +/// clients resend theirs every epoch, so two missed epochs mean the client +/// stopped using the builder, or stopped. +pub(crate) const REGISTRATION_LIFETIME_EPOCHS: Epoch = 2; + +/// Each validator's latest registration, keyed by its pubkey. +#[derive(Debug, Clone, Default)] +pub(crate) struct BuilderRegistrations(Arc>>); + +#[derive(Debug, Clone)] +struct Entry { + registration: ValidatorRegistrationV1, + /// The epoch it was last sent in. + seen: Epoch, +} + +impl BuilderRegistrations { + /// Records `registrations`, sent in `epoch`. + /// + /// One with an older timestamp than the registration held is ignored, as + /// builders ignore it; any other replaces it and restarts its lifetime. + /// Expired entries are dropped on the way, which bounds the map by the + /// registrations still being sent. + pub(crate) fn record<'a>( + &self, + registrations: impl IntoIterator, + epoch: Epoch, + ) { + let mut entries = self.0.lock().expect("builder registrations lock poisoned"); + entries.retain(|_, entry| !expired(entry, epoch)); + for registration in registrations { + match entries.get_mut(®istration.pubkey) { + Some(entry) if registration.timestamp < entry.registration.timestamp => {} + Some(entry) => { + entry.registration = registration.clone(); + entry.seen = epoch; + } + None => { + let entry = Entry { + registration: registration.clone(), + seen: epoch, + }; + entries.insert(registration.pubkey, entry); + } + } + } + } + + /// `pubkey`'s registration, if it is still current at `epoch`. + pub(crate) fn get(&self, pubkey: &BlsPubkey, epoch: Epoch) -> Option { + let entries = self.0.lock().expect("builder registrations lock poisoned"); + entries + .get(pubkey) + .filter(|entry| !expired(entry, epoch)) + .map(|entry| entry.registration.clone()) + } + + #[cfg(test)] + fn len(&self) -> usize { + self.0.lock().unwrap().len() + } +} + +fn expired(entry: &Entry, epoch: Epoch) -> bool { + epoch > entry.seen + REGISTRATION_LIFETIME_EPOCHS +} + +#[cfg(test)] +mod tests { + use ethlambda_types::beacon::primitives::ExecutionAddress; + + use super::*; + + fn registration(key: u8, timestamp: u64, gas_limit: u64) -> ValidatorRegistrationV1 { + ValidatorRegistrationV1 { + fee_recipient: ExecutionAddress::repeat_byte(key), + gas_limit, + timestamp, + pubkey: BlsPubkey([key; 48]), + } + } + + #[test] + fn a_registration_lasts_two_epochs_without_a_resend() { + let cache = BuilderRegistrations::default(); + let held = registration(1, 100, 30_000_000); + cache.record([&held], 10); + assert_eq!(cache.get(&held.pubkey, 12), Some(held.clone())); + assert_eq!(cache.get(&held.pubkey, 13), None); + } + + #[test] + fn a_resend_restarts_the_lifetime() { + let cache = BuilderRegistrations::default(); + let held = registration(1, 100, 30_000_000); + cache.record([&held], 10); + cache.record([&held], 12); + assert_eq!(cache.get(&held.pubkey, 14), Some(held)); + } + + #[test] + fn a_newer_registration_replaces_the_held_one() { + let cache = BuilderRegistrations::default(); + let old = registration(1, 100, 30_000_000); + let new = registration(1, 200, 60_000_000); + cache.record([&old], 10); + cache.record([&new], 10); + assert_eq!(cache.get(&old.pubkey, 10), Some(new)); + } + + #[test] + fn an_older_registration_is_ignored_and_does_not_refresh() { + let cache = BuilderRegistrations::default(); + let held = registration(1, 200, 60_000_000); + let stale = registration(1, 100, 30_000_000); + cache.record([&held], 10); + cache.record([&stale], 12); + assert_eq!(cache.get(&held.pubkey, 12), Some(held.clone())); + assert_eq!(cache.get(&held.pubkey, 13), None); + } + + #[test] + fn expired_entries_are_dropped_on_the_next_record() { + let cache = BuilderRegistrations::default(); + cache.record([®istration(1, 100, 1)], 10); + cache.record([®istration(2, 100, 1)], 20); + assert_eq!(cache.len(), 1); + } +} diff --git a/crates/net/rpc/src/beacon/builder_flow/mod.rs b/crates/net/rpc/src/beacon/builder_flow/mod.rs new file mode 100644 index 000000000..84138234f --- /dev/null +++ b/crates/net/rpc/src/beacon/builder_flow/mod.rs @@ -0,0 +1,5 @@ +//! The fulu builder flow's decisions: which proposers use the builder, when +//! the chain is too unhealthy to, whether a bid is acceptable, and which +//! payload goes into the block. + +pub(crate) mod cache; diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index dc6af10c5..ad509377a 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -20,6 +20,7 @@ pub(crate) mod bids; pub(crate) mod blocks; pub(crate) mod builder_client; pub(crate) mod builder_config; +pub(crate) mod builder_flow; pub(crate) mod builders; pub(crate) mod config; pub(crate) mod envelopes; From c9f4fe036931c216dec688682699f3365501e7b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:32:39 -0300 Subject: [PATCH 03/20] feat(rpc): add the builder flow's circuit breaker builder-specs requires proposers to stop using builders when the chain looks unhealthy and leaves the conditions to the client. These follow Lighthouse's defaults and read only the proposal state, where an empty slot shows as a block root repeated from the slot before. --- .../rpc/src/beacon/builder_flow/breaker.rs | 196 ++++++++++++++++++ crates/net/rpc/src/beacon/builder_flow/mod.rs | 1 + 2 files changed, 197 insertions(+) create mode 100644 crates/net/rpc/src/beacon/builder_flow/breaker.rs diff --git a/crates/net/rpc/src/beacon/builder_flow/breaker.rs b/crates/net/rpc/src/beacon/builder_flow/breaker.rs new file mode 100644 index 000000000..898e4385c --- /dev/null +++ b/crates/net/rpc/src/beacon/builder_flow/breaker.rs @@ -0,0 +1,196 @@ +//! The circuit breaker: when the chain looks unhealthy, propose a local +//! block rather than a builder's. +//! +//! builder-specs leaves the conditions to the client; these follow +//! Lighthouse's. They read the proposal state alone, so two nodes holding +//! the same state agree: a slot `i` before the proposal slot was empty iff +//! `block_roots[i] == block_roots[i - 1]`, since `process_slots` carries the +//! previous root forward over an empty slot. + +use ethlambda_state_transition::beacon::helpers::accessors::get_block_root_at_slot; +use ethlambda_types::beacon::containers::BeaconState; +use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; +use ethlambda_types::beacon::primitives::Slot; +use ethlambda_types::beacon::signing::compute_epoch_at_slot; + +/// How unhealthy the chain may look before block production skips the +/// builder. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FallbackThresholds { + /// The most consecutive empty slots allowed right before the proposal. + pub skips: u64, + /// The most empty slots allowed in the epoch's worth of slots before the + /// proposal. + pub skips_per_epoch: u64, + /// The most epochs the proposal may be past the finalized checkpoint. + pub epochs_since_finalization: u64, +} + +impl FallbackThresholds { + /// Lighthouse's defaults. + pub const DEFAULT: Self = Self { + skips: 3, + skips_per_epoch: 8, + epochs_since_finalization: 3, + }; +} + +impl Default for FallbackThresholds { + fn default() -> Self { + Self::DEFAULT + } +} + +/// Why the breaker kept the builder out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Trip { + Syncing, + OptimisticHead, + NoFinality, + ConsecutiveSkips, + SkipsPerEpoch, +} + +impl Trip { + /// The metric label. + pub(crate) fn label(self) -> &'static str { + match self { + Self::Syncing => "syncing", + Self::OptimisticHead => "optimistic_head", + Self::NoFinality => "no_finality", + Self::ConsecutiveSkips => "consecutive_skips", + Self::SkipsPerEpoch => "skips_per_epoch", + } + } +} + +/// Whether the builder may be used for a block at `state`'s slot. +/// +/// `state` is the head state advanced to the proposal slot. `optimistic` and +/// `syncing` are the node's own view, which the state cannot know. +pub(crate) fn check( + state: &BeaconState, + thresholds: &FallbackThresholds, + optimistic: bool, + syncing: bool, +) -> Result<(), Trip> { + if syncing { + return Err(Trip::Syncing); + } + if optimistic { + return Err(Trip::OptimisticHead); + } + let slot = state.slot(); + let since_finalization = + compute_epoch_at_slot(slot).saturating_sub(state.finalized_checkpoint().epoch); + if since_finalization > thresholds.epochs_since_finalization { + return Err(Trip::NoFinality); + } + let consecutive = (1..slot).rev().take_while(|&i| is_empty(state, i)).count() as u64; + if consecutive > thresholds.skips { + return Err(Trip::ConsecutiveSkips); + } + let window_start = slot.saturating_sub(SLOTS_PER_EPOCH).max(1); + let in_window = (window_start..slot).filter(|&i| is_empty(state, i)).count() as u64; + if in_window > thresholds.skips_per_epoch { + return Err(Trip::SkipsPerEpoch); + } + Ok(()) +} + +/// Whether slot `i` (with `0 < i < state.slot()`) had no block. +fn is_empty(state: &BeaconState, i: Slot) -> bool { + match ( + get_block_root_at_slot(state, i), + get_block_root_at_slot(state, i - 1), + ) { + (Ok(root), Ok(previous)) => root == previous, + // Older than the state remembers: count it as filled, so a short + // history never trips the breaker on its own. + _ => false, + } +} + +#[cfg(test)] +mod tests { + use ethlambda_state_transition::beacon::helpers::test_state::with_signing_validators_at; + use ethlambda_types::beacon::fork::ForkName; + use ethlambda_types::beacon::primitives::{Epoch, H256}; + + use super::*; + + /// A fulu state at `slot` whose every earlier slot had a block except + /// those in `empty`, finalized at `finalized_epoch`. + fn state_at(slot: Slot, empty: &[Slot], finalized_epoch: Epoch) -> BeaconState { + let mut state = with_signing_validators_at(ForkName::Fulu, 8); + let BeaconState::Fulu(fulu) = &mut state else { + unreachable!("built as fulu") + }; + fulu.slot = slot; + fulu.finalized_checkpoint.epoch = finalized_epoch; + let history = fulu.block_roots.len() as u64; + let mut root = H256([0; 32]); + for i in 0..slot { + if !empty.contains(&i) { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&(i + 1).to_le_bytes()); + root = H256(bytes); + } + fulu.block_roots[(i % history) as usize] = root; + } + state + } + + const LENIENT_SYNC: (bool, bool) = (false, false); + + fn check_at(state: &BeaconState) -> Result<(), Trip> { + let (optimistic, syncing) = LENIENT_SYNC; + check(state, &FallbackThresholds::DEFAULT, optimistic, syncing) + } + + #[test] + fn a_healthy_chain_uses_the_builder() { + assert_eq!(check_at(&state_at(100, &[], 2)), Ok(())); + } + + #[test] + fn three_empty_slots_in_a_row_are_tolerated_and_four_are_not() { + assert_eq!(check_at(&state_at(100, &[97, 98, 99], 2)), Ok(())); + assert_eq!( + check_at(&state_at(100, &[96, 97, 98, 99], 2)), + Err(Trip::ConsecutiveSkips) + ); + } + + #[test] + fn eight_scattered_empty_slots_in_an_epoch_are_tolerated_and_nine_are_not() { + let eight = [70, 72, 74, 76, 78, 80, 82, 84]; + assert_eq!(check_at(&state_at(100, &eight, 2)), Ok(())); + let nine = [70, 72, 74, 76, 78, 80, 82, 84, 86]; + assert_eq!(check_at(&state_at(100, &nine, 2)), Err(Trip::SkipsPerEpoch)); + } + + #[test] + fn empty_slots_outside_the_epoch_window_do_not_count() { + let old = [10, 12, 14, 16, 18, 20, 22, 24, 26, 28]; + assert_eq!(check_at(&state_at(100, &old, 2)), Ok(())); + } + + #[test] + fn three_epochs_without_finality_are_tolerated_and_four_are_not() { + // Slot 200 is in epoch 6. + assert_eq!(check_at(&state_at(200, &[], 3)), Ok(())); + assert_eq!(check_at(&state_at(200, &[], 2)), Err(Trip::NoFinality)); + } + + #[test] + fn an_optimistic_head_or_a_syncing_node_keeps_the_builder_out() { + let state = state_at(100, &[], 2); + let thresholds = FallbackThresholds::DEFAULT; + assert_eq!( + check(&state, &thresholds, true, false), + Err(Trip::OptimisticHead) + ); + assert_eq!(check(&state, &thresholds, false, true), Err(Trip::Syncing)); + } +} diff --git a/crates/net/rpc/src/beacon/builder_flow/mod.rs b/crates/net/rpc/src/beacon/builder_flow/mod.rs index 84138234f..40587048f 100644 --- a/crates/net/rpc/src/beacon/builder_flow/mod.rs +++ b/crates/net/rpc/src/beacon/builder_flow/mod.rs @@ -2,4 +2,5 @@ //! the chain is too unhealthy to, whether a bid is acceptable, and which //! payload goes into the block. +pub(crate) mod breaker; pub(crate) mod cache; From 87a9851f67090785fe57456d258ad1b9b8deb457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:34:28 -0300 Subject: [PATCH 04/20] feat(rpc): validate a builder's bid before it competes A bid must pay something, sit on the proposal's parent with its prev_randao, timestamp and withdrawals, carry blob gas for exactly its blobs, step its gas limit toward the proposer's registered target, and be signed by its builder. Each refusal has its own reason so the metric can tell a misbehaving relay from a misconfigured one. The gas rule is gloas's gossip predicate, pinned to Lighthouse's vectors. --- .../src/beacon/builder_client/mock_relay.rs | 26 +- crates/net/rpc/src/beacon/builder_flow/bid.rs | 247 ++++++++++++++++++ crates/net/rpc/src/beacon/builder_flow/mod.rs | 1 + 3 files changed, 266 insertions(+), 8 deletions(-) create mode 100644 crates/net/rpc/src/beacon/builder_flow/bid.rs diff --git a/crates/net/rpc/src/beacon/builder_client/mock_relay.rs b/crates/net/rpc/src/beacon/builder_client/mock_relay.rs index 92c6d2281..219ce3f89 100644 --- a/crates/net/rpc/src/beacon/builder_client/mock_relay.rs +++ b/crates/net/rpc/src/beacon/builder_client/mock_relay.rs @@ -187,15 +187,25 @@ impl MockRelay { /// `message` as this relay sends it: naming the relay's key and signed /// with it under the builder domain. - pub(crate) fn sign_bid(&self, mut message: BuilderBid) -> SignedBuilderBid { - message.pubkey = self.pubkey(); + pub(crate) fn sign_bid(&self, message: BuilderBid) -> SignedBuilderBid { let genesis_fork_version = self.state.lock().unwrap().genesis_fork_version; - let domain = compute_builder_domain(genesis_fork_version); - let root = compute_signing_root(message.hash_tree_root(), domain); - SignedBuilderBid { - message, - signature: sign_for(RELAY_KEY, root), - } + signed_bid(message, genesis_fork_version) + } +} + +/// `message` signed by the mock relay's key under the builder domain of the +/// chain whose genesis fork version is `genesis_fork_version`, naming that +/// key. +pub(crate) fn signed_bid( + mut message: BuilderBid, + genesis_fork_version: Version, +) -> SignedBuilderBid { + message.pubkey = BlsPubkey(secret_key_for(RELAY_KEY).sk_to_pk().to_bytes()); + let domain = compute_builder_domain(genesis_fork_version); + let root = compute_signing_root(message.hash_tree_root(), domain); + SignedBuilderBid { + message, + signature: sign_for(RELAY_KEY, root), } } diff --git a/crates/net/rpc/src/beacon/builder_flow/bid.rs b/crates/net/rpc/src/beacon/builder_flow/bid.rs new file mode 100644 index 000000000..7e047815c --- /dev/null +++ b/crates/net/rpc/src/beacon/builder_flow/bid.rs @@ -0,0 +1,247 @@ +//! Checks on a builder's bid before it may stand against the local payload. +//! +//! Everything that can be checked without the payload itself. The header +//! fields the state transition checks (parent hash, prev_randao, timestamp, +//! withdrawals, blob count) are checked here too, before +//! `assemble_blinded_block` repeats them as a backstop, so a refusal has a +//! reason to count and costs no state transition. What is left unchecked +//! (the payload's execution validity, its block hash and the fields only an +//! execution client could verify) is trusted to the relay, as every client +//! does: the payload is revealed only after the proposer signs. + +use ethlambda_state_transition::beacon::block_production::PayloadInputs; +use ethlambda_state_transition::beacon::bls; +use ethlambda_state_transition::beacon::gossip::execution_payload_bid::is_gas_limit_target_compatible; +use ethlambda_types::beacon::builder::SignedBuilderBid; +use ethlambda_types::beacon::containers::capella; +use ethlambda_types::beacon::primitives::{HashTreeRoot as _, Version}; +use ethlambda_types::beacon::signing::{compute_builder_domain, compute_signing_root}; + +use crate::beacon::bid_selection::wei_u128; + +/// EIP-4844's blob gas per blob. +pub(crate) const GAS_PER_BLOB: u64 = 1 << 17; + +/// What a bid is checked against, all from the proposal state and the +/// proposer's registration. +pub(crate) struct BidContext<'a> { + pub(crate) genesis_fork_version: Version, + pub(crate) inputs: &'a PayloadInputs, + /// The gas limit of the payload the block builds on. + pub(crate) parent_gas_limit: u64, + /// The gas limit the proposer registered. + pub(crate) target_gas_limit: u64, + pub(crate) max_blobs: u64, +} + +/// Why a bid was refused. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Rejection { + ZeroValue, + ParentHash, + PrevRandao, + Timestamp, + Withdrawals, + TooManyBlobs, + BlobGas, + GasLimit, + Signature, +} + +impl Rejection { + /// The metric label. + pub(crate) fn label(self) -> &'static str { + match self { + Self::ZeroValue => "zero_value", + Self::ParentHash => "parent_hash", + Self::PrevRandao => "prev_randao", + Self::Timestamp => "timestamp", + Self::Withdrawals => "withdrawals", + Self::TooManyBlobs => "too_many_blobs", + Self::BlobGas => "blob_gas", + Self::GasLimit => "gas_limit", + Self::Signature => "signature", + } + } +} + +/// Whether `bid` is acceptable for the proposal `context` describes. +/// Cheap checks first; the signature, the one that costs a pairing, last. +pub(crate) fn validate(bid: &SignedBuilderBid, context: &BidContext) -> Result<(), Rejection> { + let message = &bid.message; + let header = &message.header; + if wei_u128(&message.value) == 0 { + return Err(Rejection::ZeroValue); + } + if header.parent_hash != context.inputs.parent_hash { + return Err(Rejection::ParentHash); + } + if header.prev_randao != context.inputs.prev_randao { + return Err(Rejection::PrevRandao); + } + if header.timestamp != context.inputs.timestamp { + return Err(Rejection::Timestamp); + } + let withdrawals: capella::Withdrawals = context + .inputs + .withdrawals + .clone() + .try_into() + .map_err(|_| Rejection::Withdrawals)?; + if header.withdrawals_root != withdrawals.hash_tree_root() { + return Err(Rejection::Withdrawals); + } + let blobs = message.blob_kzg_commitments.len() as u64; + if blobs > context.max_blobs { + return Err(Rejection::TooManyBlobs); + } + if header.blob_gas_used != blobs * GAS_PER_BLOB { + return Err(Rejection::BlobGas); + } + if !is_gas_limit_target_compatible( + context.parent_gas_limit, + header.gas_limit, + context.target_gas_limit, + ) { + return Err(Rejection::GasLimit); + } + let domain = compute_builder_domain(context.genesis_fork_version); + let root = compute_signing_root(message.hash_tree_root(), domain); + if !bls::verify(&message.pubkey, root, &bid.signature) { + return Err(Rejection::Signature); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use ethlambda_types::beacon::builder::BuilderBid; + use ethlambda_types::beacon::containers::capella::Withdrawal; + use ethlambda_types::beacon::containers::deneb::ExecutionPayloadHeader; + use ethlambda_types::beacon::containers::electra::{self, ExecutionRequests}; + use ethlambda_types::beacon::primitives::{ + BlsPubkey, Bytes32, ExecutionAddress, ExecutionBlockHash, KzgCommitment, Uint256, + }; + + use super::*; + use crate::beacon::builder_client::mock_relay::signed_bid; + + const GENESIS_FORK_VERSION: Version = [0x10, 0x00, 0x00, 0x38]; + const PARENT_GAS_LIMIT: u64 = 30_000_000; + const TARGET_GAS_LIMIT: u64 = 40_000_000; + /// Lighthouse's `expected_gas_limit(30M, 40M)`: one adjustment step up. + const NEXT_GAS_LIMIT: u64 = 30_029_295; + + fn inputs() -> PayloadInputs { + PayloadInputs { + timestamp: 1_700_000_012, + prev_randao: Bytes32::repeat_byte(2), + withdrawals: vec![Withdrawal { + index: 1, + validator_index: 2, + address: ExecutionAddress::repeat_byte(4), + amount: 5, + }], + parent_hash: ExecutionBlockHash::repeat_byte(3), + } + } + + fn context(inputs: &PayloadInputs) -> BidContext<'_> { + BidContext { + genesis_fork_version: GENESIS_FORK_VERSION, + inputs, + parent_gas_limit: PARENT_GAS_LIMIT, + target_gas_limit: TARGET_GAS_LIMIT, + max_blobs: 9, + } + } + + /// A one-blob bid that passes every check against `inputs()`, with `edit` + /// applied before it is signed. + fn bid(edit: impl FnOnce(&mut BuilderBid)) -> SignedBuilderBid { + let inputs = inputs(); + let withdrawals: capella::Withdrawals = inputs.withdrawals.clone().try_into().unwrap(); + let mut header = + ExecutionPayloadHeader::from(&electra::BeaconBlockBody::empty().execution_payload); + header.parent_hash = inputs.parent_hash; + header.prev_randao = inputs.prev_randao; + header.timestamp = inputs.timestamp; + header.withdrawals_root = withdrawals.hash_tree_root(); + header.gas_limit = NEXT_GAS_LIMIT; + header.blob_gas_used = GAS_PER_BLOB; + let mut message = BuilderBid { + header, + blob_kzg_commitments: vec![KzgCommitment::default()].try_into().unwrap(), + execution_requests: ExecutionRequests::default(), + value: Uint256::from(1_000_000_000u64), + pubkey: BlsPubkey([0; 48]), + }; + edit(&mut message); + signed_bid(message, GENESIS_FORK_VERSION) + } + + fn rejection(bid: &SignedBuilderBid) -> Option { + validate(bid, &context(&inputs())).err() + } + + #[test] + fn a_bid_that_matches_the_proposal_is_accepted() { + assert_eq!(rejection(&bid(|_| {})), None); + } + + #[test] + fn each_check_refuses_its_own_field() { + type Edit = fn(&mut BuilderBid); + let cases: [(Rejection, Edit); 8] = [ + (Rejection::ZeroValue, |bid| bid.value = Uint256::from(0u64)), + (Rejection::ParentHash, |bid| { + bid.header.parent_hash = ExecutionBlockHash::repeat_byte(9) + }), + (Rejection::PrevRandao, |bid| { + bid.header.prev_randao = Bytes32::repeat_byte(9) + }), + (Rejection::Timestamp, |bid| bid.header.timestamp += 1), + (Rejection::Withdrawals, |bid| { + bid.header.withdrawals_root = Bytes32::repeat_byte(9) + }), + (Rejection::TooManyBlobs, |bid| { + bid.blob_kzg_commitments = vec![KzgCommitment::default(); 10].try_into().unwrap(); + bid.header.blob_gas_used = 10 * GAS_PER_BLOB; + }), + (Rejection::BlobGas, |bid| bid.header.blob_gas_used = 0), + (Rejection::GasLimit, |bid| { + bid.header.gas_limit = TARGET_GAS_LIMIT + }), + ]; + for (expected, edit) in cases { + assert_eq!(rejection(&bid(edit)), Some(expected), "{expected:?}"); + } + } + + #[test] + fn a_bid_signed_by_another_key_is_refused() { + let mut forged = bid(|_| {}); + forged.message.value = Uint256::from(2_000_000_000u64); + assert_eq!(rejection(&forged), Some(Rejection::Signature)); + } + + /// The gloas gossip predicate this reuses agrees with Lighthouse's + /// `expected_gas_limit` on its own test vectors. + #[test] + fn the_gas_limit_rule_matches_lighthouses_vectors() { + let vectors = [ + (30_000_000, 30_000_000, 30_000_000), + (30_000_000, 40_000_000, 30_029_295), + (30_029_295, 40_000_000, 30_058_619), + (30_058_619, 30_000_000, 30_029_266), + ]; + for (parent, target, expected) in vectors { + assert!(is_gas_limit_target_compatible(parent, expected, target)); + assert!(!is_gas_limit_target_compatible( + parent, + expected + 1, + target + )); + } + } +} diff --git a/crates/net/rpc/src/beacon/builder_flow/mod.rs b/crates/net/rpc/src/beacon/builder_flow/mod.rs index 40587048f..2787a7cd6 100644 --- a/crates/net/rpc/src/beacon/builder_flow/mod.rs +++ b/crates/net/rpc/src/beacon/builder_flow/mod.rs @@ -2,5 +2,6 @@ //! the chain is too unhealthy to, whether a bid is acceptable, and which //! payload goes into the block. +pub(crate) mod bid; pub(crate) mod breaker; pub(crate) mod cache; From 15bc10e6812fe7f5e0a893f69aa3308e22d7296f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:35:17 -0300 Subject: [PATCH 05/20] feat(rpc): choose between the local payload and a builder's beacon-APIs' builder_boost_factor rule taken literally, with the execution client's shouldOverrideBuilder above it and a tie going to the local payload, which never depends on a relay to reveal it. --- .../net/rpc/src/beacon/builder_flow/choice.rs | 146 ++++++++++++++++++ crates/net/rpc/src/beacon/builder_flow/mod.rs | 1 + 2 files changed, 147 insertions(+) create mode 100644 crates/net/rpc/src/beacon/builder_flow/choice.rs diff --git a/crates/net/rpc/src/beacon/builder_flow/choice.rs b/crates/net/rpc/src/beacon/builder_flow/choice.rs new file mode 100644 index 000000000..7d76ca27c --- /dev/null +++ b/crates/net/rpc/src/beacon/builder_flow/choice.rs @@ -0,0 +1,146 @@ +//! Choosing between the local payload and a builder's. + +/// beacon-APIs' default `builder_boost_factor`: compare the two as they are. +pub(crate) const DEFAULT_BOOST_FACTOR: u64 = 100; + +/// Where the block's payload comes from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Source { + Local, + Builder, +} + +impl Source { + /// The metric label. + pub(crate) fn label(self) -> &'static str { + match self { + Self::Local => "local", + Self::Builder => "builder", + } + } +} + +/// The local payload's side of the comparison. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct LocalOffer { + /// What it pays, in wei. + pub(crate) value: u128, + /// The execution client wants it built locally whatever a bid pays. + pub(crate) should_override_builder: bool, +} + +/// Which payload the block carries, or `None` when neither is usable. +/// +/// beacon-APIs' `builder_boost_factor`, taken literally: the local payload +/// wins iff `local >= factor * (bid / 100)`. So 100 compares the two as they +/// are, 0 always keeps the local payload, and a tie goes local. +/// `u64::MAX` always takes the bid. The execution client's +/// `shouldOverrideBuilder` beats any bid. When one side failed, the other +/// stands. +pub(crate) fn choose( + local: Option, + bid_value: Option, + boost_factor: u64, +) -> Option { + match (local, bid_value) { + (None, None) => None, + (Some(_), None) => Some(Source::Local), + (None, Some(_)) => Some(Source::Builder), + (Some(local), Some(_)) if local.should_override_builder => Some(Source::Local), + (Some(_), Some(_)) if boost_factor == u64::MAX => Some(Source::Builder), + (Some(local), Some(bid)) => { + let boosted = u128::from(boost_factor).saturating_mul(bid / 100); + Some(if local.value >= boosted { + Source::Local + } else { + Source::Builder + }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ETH: u128 = 1_000_000_000_000_000_000; + + fn local(value: u128) -> Option { + Some(LocalOffer { + value, + should_override_builder: false, + }) + } + + #[test] + fn the_selection_table() { + let overriding = Some(LocalOffer { + value: 1, + should_override_builder: true, + }); + let cases = [ + ("neither", None, None, 100, None), + ("local only", local(1), None, 100, Some(Source::Local)), + ("bid only", None, Some(ETH), 100, Some(Source::Builder)), + ( + "bid pays more", + local(ETH), + Some(2 * ETH), + 100, + Some(Source::Builder), + ), + ( + "local pays more", + local(2 * ETH), + Some(ETH), + 100, + Some(Source::Local), + ), + ( + "a tie goes local", + local(ETH), + Some(ETH), + 100, + Some(Source::Local), + ), + ( + "factor 0 keeps local", + local(1), + Some(100 * ETH), + 0, + Some(Source::Local), + ), + ( + "factor 50 halves the bid", + local(ETH), + Some(2 * ETH), + 50, + Some(Source::Local), + ), + ( + "factor 200 doubles it", + local(ETH + 1), + Some(ETH), + 200, + Some(Source::Builder), + ), + ( + "u64::MAX takes the bid", + local(100 * ETH), + Some(1), + u64::MAX, + Some(Source::Builder), + ), + ( + "the override keeps local", + overriding, + Some(100 * ETH), + u64::MAX, + Some(Source::Local), + ), + ]; + for (name, local, bid, factor, expected) in cases { + assert_eq!(choose(local, bid, factor), expected, "{name}"); + } + } +} diff --git a/crates/net/rpc/src/beacon/builder_flow/mod.rs b/crates/net/rpc/src/beacon/builder_flow/mod.rs index 2787a7cd6..fea0679b0 100644 --- a/crates/net/rpc/src/beacon/builder_flow/mod.rs +++ b/crates/net/rpc/src/beacon/builder_flow/mod.rs @@ -5,3 +5,4 @@ pub(crate) mod bid; pub(crate) mod breaker; pub(crate) mod cache; +pub(crate) mod choice; From 52a6b849d1f7bdae080d2bced6176492681fd702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:38:26 -0300 Subject: [PATCH 06/20] feat(rpc): share the builder's state and remember registrations The builder client, the registrations it was sent, the circuit breaker's thresholds and the sync status travel together, layered only when --builder is set, so every handler and test router without one keeps the local path untouched. register_validator now records what it forwards, which block production reads next. The breaker's thresholds are flags, defaulting to Lighthouse's. --- bin/ethlambda/src/cli.rs | 39 ++++++ bin/ethlambda/src/main.rs | 9 ++ crates/net/rpc/src/beacon/builder_flow/mod.rs | 33 +++++ crates/net/rpc/src/beacon/registrations.rs | 116 +++++++++++++----- crates/net/rpc/src/lib.rs | 18 ++- docs/cli.md | 3 + 6 files changed, 185 insertions(+), 33 deletions(-) diff --git a/bin/ethlambda/src/cli.rs b/bin/ethlambda/src/cli.rs index 4101c56a5..72ab02014 100644 --- a/bin/ethlambda/src/cli.rs +++ b/bin/ethlambda/src/cli.rs @@ -325,6 +325,21 @@ pub(crate) struct MainnetOptions { value_parser = clap::value_parser!(u64).range(1..), )] pub(crate) builder_header_timeout: u64, + + /// Build locally, skipping the builder, after more than this many + /// consecutive empty slots right before the proposal (the circuit breaker). + #[arg(long, default_value_t = ethlambda_rpc::FallbackThresholds::DEFAULT.skips, requires = "builder")] + pub(crate) builder_fallback_skips: u64, + + /// Build locally after more than this many empty slots in the epoch's + /// worth of slots before the proposal. + #[arg(long, default_value_t = ethlambda_rpc::FallbackThresholds::DEFAULT.skips_per_epoch, requires = "builder")] + pub(crate) builder_fallback_skips_per_epoch: u64, + + /// Build locally when the proposal is more than this many epochs past the + /// finalized checkpoint. + #[arg(long, default_value_t = ethlambda_rpc::FallbackThresholds::DEFAULT.epochs_since_finalization, requires = "builder")] + pub(crate) builder_fallback_epochs_since_finalization: u64, } /// Flags only the lean chain takes. @@ -1067,6 +1082,30 @@ mod tests { } } + #[test] + fn the_fallback_flags_parse_and_default_to_lighthouses() { + let beacon = parse_beacon(beacon_args()); + assert_eq!(beacon.mainnet.builder_fallback_skips, 3); + assert_eq!(beacon.mainnet.builder_fallback_skips_per_epoch, 8); + assert_eq!(beacon.mainnet.builder_fallback_epochs_since_finalization, 3); + + let mut args = beacon_args(); + args.extend([ + "--builder", + "http://127.0.0.1:18550", + "--builder-fallback-skips", + "5", + "--builder-fallback-skips-per-epoch", + "12", + "--builder-fallback-epochs-since-finalization", + "2", + ]); + let beacon = parse_beacon(args); + assert_eq!(beacon.mainnet.builder_fallback_skips, 5); + assert_eq!(beacon.mainnet.builder_fallback_skips_per_epoch, 12); + assert_eq!(beacon.mainnet.builder_fallback_epochs_since_finalization, 2); + } + #[test] fn a_zero_header_timeout_is_refused() { let mut args = beacon_args(); diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 813055209..5f76f9c52 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -465,6 +465,14 @@ async fn run_node(options: Options) -> eyre::Result<()> { }) .transpose()?, }; + let builder_fallback = match &network { + Network::Lean(_) => ethlambda_rpc::FallbackThresholds::DEFAULT, + Network::Mainnet { mainnet, .. } => ethlambda_rpc::FallbackThresholds { + skips: mainnet.builder_fallback_skips, + skips_per_epoch: mainnet.builder_fallback_skips_per_epoch, + epochs_since_finalization: mainnet.builder_fallback_epochs_since_finalization, + }, + }; if let Some(builder) = builder.clone() { tokio::spawn(async move { let _ = builder @@ -872,6 +880,7 @@ async fn run_node(options: Options) -> eyre::Result<()> { client_version: version::engine_client_version(), identity: rpc_identity, builder, + builder_fallback, }, local_peer_id, rpc_shutdown, diff --git a/crates/net/rpc/src/beacon/builder_flow/mod.rs b/crates/net/rpc/src/beacon/builder_flow/mod.rs index fea0679b0..6fc76ae4e 100644 --- a/crates/net/rpc/src/beacon/builder_flow/mod.rs +++ b/crates/net/rpc/src/beacon/builder_flow/mod.rs @@ -6,3 +6,36 @@ pub(crate) mod bid; pub(crate) mod breaker; pub(crate) mod cache; pub(crate) mod choice; + +use ethlambda_blockchain::SyncStatusController; + +use super::builder_client::BuilderClient; +use breaker::FallbackThresholds; +use cache::BuilderRegistrations; + +/// What the builder flow's handlers share. It is layered only when +/// `--builder` is set, so a handler reads it as +/// `Option>>`. A node without a builder, or a test +/// router that never layers it, takes the local path. +pub(crate) struct Builder { + pub(crate) client: BuilderClient, + pub(crate) registrations: BuilderRegistrations, + pub(crate) fallback: FallbackThresholds, + /// The node's sync status, which the circuit breaker reads. + pub(crate) sync_status: SyncStatusController, +} + +impl Builder { + pub(crate) fn new( + client: BuilderClient, + fallback: FallbackThresholds, + sync_status: SyncStatusController, + ) -> Self { + Self { + client, + registrations: BuilderRegistrations::default(), + fallback, + sync_status, + } + } +} diff --git a/crates/net/rpc/src/beacon/registrations.rs b/crates/net/rpc/src/beacon/registrations.rs index bca74807e..f90124571 100644 --- a/crates/net/rpc/src/beacon/registrations.rs +++ b/crates/net/rpc/src/beacon/registrations.rs @@ -9,9 +9,12 @@ //! not reach the builder, not that the request fails. Each kept //! registration also becomes its validator's fee recipient, as //! `prepare_beacon_proposer` would make it, so a block built locally after -//! all still pays the address the validator registered. +//! all still pays the address the validator registered. The kept +//! registrations are also remembered for block production, which asks the +//! builder only for proposers that registered. use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use axum::body::Bytes; use axum::extract::{DefaultBodyLimit, State}; @@ -29,7 +32,8 @@ use ethlambda_types::beacon::signing::compute_epoch_at_slot; use libssz::SszDecode as _; use tracing::warn; -use super::builder_client::{BuilderClient, BuilderError}; +use super::builder_client::BuilderError; +use super::builder_flow::Builder; use super::validator::{FeeRecipients, head}; use super::{ApiError, BodyEncoding}; use crate::metrics::inc_builder_registrations; @@ -45,11 +49,12 @@ pub(crate) fn routes() -> Router { async fn post_register_validator( State(store): State, - Extension(builder): Extension>, + builder: Option>>, Extension(fee_recipients): Extension, headers: HeaderMap, body: Bytes, ) -> Response { + let builder = builder.map(|Extension(builder)| builder); register(store, builder, fee_recipients, &headers, &body) .await .map_or_else(IntoResponse::into_response, |()| { @@ -59,7 +64,7 @@ async fn post_register_validator( async fn register( store: Store, - builder: Option, + builder: Option>, fee_recipients: FeeRecipients, headers: &HeaderMap, body: &[u8], @@ -72,12 +77,13 @@ async fn register( // The head-state read and the walk over the whole registry are blocking // work, as the duty endpoints' walks are. let submitted = registrations.len(); - let kept = tokio::task::spawn_blocking(move || { + let (kept, epoch) = tokio::task::spawn_blocking(move || { let (_, state) = head(&store)?; // Judged at the later of the head's epoch and the wall clock's, so a // lagging head cannot let an exited validator through. let wall_epoch = compute_epoch_at_slot(super::node::wall_slot(&store)); - Ok::<_, ApiError>(active_or_pending(&state, registrations, wall_epoch)) + let epoch = compute_epoch_at_slot(state.slot()).max(wall_epoch); + Ok::<_, ApiError>((active_or_pending(&state, registrations, epoch), epoch)) }) .await .map_err(|_| ApiError::Internal("checking the registrations failed"))??; @@ -92,6 +98,12 @@ async fn register( recipients.insert(*index, registration.message.fee_recipient); } } + // Remembered at the epoch the filter judged them at, so block production + // sees them for as long as the validator client keeps sending them. + builder.registrations.record( + kept.iter().map(|(_, registration)| ®istration.message), + epoch, + ); let kept: Vec = kept .into_iter() .map(|(_, registration)| registration) @@ -100,7 +112,7 @@ async fn register( // and the forward, its count and its log must complete regardless. let forwarded = tokio::spawn(async move { let count = kept.len() as u64; - let result = builder.register_validators(&kept).await; + let result = builder.client.register_validators(&kept).await; match &result { Ok(()) => inc_builder_registrations("forwarded", count), Err(err) => { @@ -133,14 +145,13 @@ fn decode( } /// The registrations whose validator `state` holds as active or pending at -/// the later of its own epoch and `wall_epoch`, each with its validator -/// index. One pass over the registry, whatever the number of registrations. +/// `epoch`, each with its validator index. One pass over the registry, +/// whatever the number of registrations. fn active_or_pending( state: &BeaconState, registrations: Vec, - wall_epoch: Epoch, + epoch: Epoch, ) -> Vec<(ValidatorIndex, SignedValidatorRegistrationV1)> { - let epoch = compute_epoch_at_slot(state.slot()).max(wall_epoch); let wanted: HashSet = registrations .iter() .map(|registration| registration.message.pubkey) @@ -168,6 +179,8 @@ mod tests { use axum::body::Body; use axum::http::{Request, StatusCode}; + use ethlambda_blockchain::SyncStatusController; + use ethlambda_blockchain::metrics::SyncStatus; use ethlambda_state_transition::beacon::helpers::test_state::with_signing_validators_at; use ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH; use ethlambda_types::beacon::fork::ForkName; @@ -177,6 +190,8 @@ mod tests { use tower::ServiceExt as _; use super::*; + use crate::FallbackThresholds; + use crate::beacon::builder_client::BuilderClient; use crate::beacon::builder_client::mock_relay::{MockRelay, signed_registration}; use crate::test_utils::beacon_store_at; @@ -202,14 +217,15 @@ mod tests { async fn post( store: Store, - builder: Option, + builder: Option>, fee_recipients: FeeRecipients, request: Request, ) -> (StatusCode, String) { - let app = routes() - .with_state(store) - .layer(Extension(builder)) - .layer(Extension(fee_recipients)); + let app = routes().with_state(store).layer(Extension(fee_recipients)); + let app = match builder { + Some(builder) => app.layer(Extension(builder)), + None => app, + }; let response = app.oneshot(request).await.unwrap(); let status = response.status(); let body = response.into_body().collect().await.unwrap().to_bytes(); @@ -223,11 +239,19 @@ mod tests { .unwrap() } - fn client_for(relay: &MockRelay) -> Option { - Some( + fn builder_for(client: BuilderClient) -> Arc { + Arc::new(Builder::new( + client, + FallbackThresholds::DEFAULT, + SyncStatusController::new(SyncStatus::Synced), + )) + } + + fn client_for(relay: &MockRelay) -> Option> { + let client = BuilderClient::new(relay.url.clone(), Duration::from_secs(1), "ethlambda/test") - .unwrap(), - ) + .unwrap(); + Some(builder_for(client)) } #[tokio::test] @@ -338,12 +362,14 @@ mod tests { async fn an_unreachable_builder_is_a_server_error() { let store = store_with_exited(&[]); let registration = signed_registration(0, genesis_fork_version(&store)); - let builder = BuilderClient::new( - "http://127.0.0.1:1", - Duration::from_secs(1), - "ethlambda/test", - ) - .unwrap(); + let builder = builder_for( + BuilderClient::new( + "http://127.0.0.1:1", + Duration::from_secs(1), + "ethlambda/test", + ) + .unwrap(), + ); let (status, _) = post( store, Some(builder), @@ -428,12 +454,14 @@ mod tests { async fn fee_recipients_are_set_even_when_the_builder_is_unreachable() { let store = store_with_exited(&[]); let registration = signed_registration(0, genesis_fork_version(&store)); - let builder = BuilderClient::new( - "http://127.0.0.1:1", - Duration::from_secs(1), - "ethlambda/test", - ) - .unwrap(); + let builder = builder_for( + BuilderClient::new( + "http://127.0.0.1:1", + Duration::from_secs(1), + "ethlambda/test", + ) + .unwrap(), + ); let fee_recipients = FeeRecipients::default(); let (status, _) = post( @@ -451,6 +479,30 @@ mod tests { ); } + #[tokio::test] + async fn kept_registrations_are_remembered_for_block_production() { + let store = store_with_exited(&[]); + let relay = MockRelay::start(genesis_fork_version(&store)).await; + let builder = client_for(&relay).unwrap(); + let kept = signed_registration(0, genesis_fork_version(&store)); + let unknown = signed_registration(VALIDATORS + 10, genesis_fork_version(&store)); + + let (status, body) = post( + store, + Some(builder.clone()), + FeeRecipients::default(), + json_request(&[kept.clone(), unknown.clone()]), + ) + .await; + + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + builder.registrations.get(&kept.message.pubkey, 0), + Some(kept.message) + ); + assert_eq!(builder.registrations.get(&unknown.message.pubkey, 0), None); + } + #[tokio::test] async fn a_malformed_body_is_a_400() { let store = store_with_exited(&[]); diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index af3f5f9cc..42dc58a1f 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -30,6 +30,7 @@ mod spec; pub mod test_driver; pub use beacon::builder_client::{BuilderClient, BuilderError, VersionedBid}; +pub use beacon::builder_flow::breaker::FallbackThresholds; pub(crate) use base::json_response; @@ -245,6 +246,9 @@ pub struct BeaconApiHandles { /// The builder `--builder` names, for the builder flow's endpoints. /// `None` when the node runs without one. pub builder: Option, + /// When block production skips the builder for a local block + /// (`--builder-fallback-*`). Unused without a builder. + pub builder_fallback: FallbackThresholds, } /// Start the HTTP servers for a beacon node. @@ -261,6 +265,13 @@ pub async fn start_beacon_rpc_server( peer_id: String, shutdown: CancellationToken, ) -> Result<(), std::io::Error> { + let builder = handles.builder.map(|client| { + Arc::new(beacon::builder_flow::Builder::new( + client, + handles.builder_fallback, + sync_status.clone(), + )) + }); let api_router = build_beacon_api_router(store, config.version, peer_id) .layer(Extension(sync_status)) .layer(Extension(handles.p2p)) @@ -270,12 +281,17 @@ pub async fn start_beacon_rpc_server( .layer(Extension(handles.custody_columns)) .layer(Extension(beacon::validator::FeeRecipients::default())) .layer(Extension(handles.engine)) - .layer(Extension(handles.builder)) .layer(Extension(handles.events)) .layer(Extension(handles.identity)) .layer(Extension(beacon::graffiti::OwnVersion(Arc::new( handles.client_version, )))); + // Layered only when `--builder` is set, so every handler reads it as an + // `Option>` and a node without one keeps the local path. + let api_router = match builder { + Some(builder) => api_router.layer(Extension(builder)), + None => api_router, + }; start_http_servers(config, Some(api_router), shutdown).await } diff --git a/docs/cli.md b/docs/cli.md index 489b24f9e..b9a65aa5d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -116,6 +116,9 @@ from a normal build. | `--safe-slots-to-import-optimistically` | the specification's own value | How far behind the wall clock a block must be before it may be imported optimistically on age alone | | `--builder` | none | Base URL of a builder API endpoint, normally MEV-Boost (e.g. `http://127.0.0.1:18550`). Without it every block is built locally | | `--builder-header-timeout` | `1000` | Milliseconds block production waits for the builder's bid; also sent to the builder as `X-Timeout-Ms`. Requires `--builder` | +| `--builder-fallback-skips` | `3` | Build locally when more than this many consecutive slots right before the proposal had no block (the circuit breaker). Requires `--builder` | +| `--builder-fallback-skips-per-epoch` | `8` | Build locally when more than this many of the slots in the epoch's worth before the proposal had no block. Requires `--builder` | +| `--builder-fallback-epochs-since-finalization` | `3` | Build locally when the proposal is more than this many epochs past the finalized checkpoint. Requires `--builder` | Accepted range is `CUSTODY_REQUIREMENT` to `NUMBER_OF_CUSTODY_GROUPS` (4 to 128), enforced at parse time rather than clamped: serving a different set than From 0731823d3abc862399a9e89963c22f9453cd4229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:43:37 -0300 Subject: [PATCH 07/20] feat(rpc): serve a builder's blinded block from produceBlockV3 With a builder configured, the proposer registered and the circuit breaker quiet, the bid is fetched alongside the local payload. A bid that validates is built into a blinded block, the state root computed by assemble_blinded_block, and wins if it pays more after the caller's builder_boost_factor; the execution client's shouldOverrideBuilder keeps the local payload whatever the bid. Either side failing leaves the other, and with no builder in play nothing changes. --- crates/net/rpc/src/beacon/proposal.rs | 714 ++++++++++++++++-- .../rpc/src/beacon/validator_client_tests.rs | 19 +- crates/net/rpc/src/metrics.rs | 48 ++ 3 files changed, 702 insertions(+), 79 deletions(-) diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index 8dcd6115d..a582394a1 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -3,9 +3,10 @@ //! execution client, and `POST /eth/v2/beacon/blocks` (`publishBlockV2`), the //! signed block back to gossip and import. //! -//! Only unblinded, locally built blocks: this node has no builder flow, so -//! `builder_boost_factor` is accepted and ignored, and every answer carries -//! `Eth-Execution-Payload-Blinded: false`. +//! Locally built blocks, or, with `--builder` set and a builder's bid that +//! validates and beats the local payload under `builder_boost_factor`, the +//! blinded block around that bid. Every answer says which in +//! `Eth-Execution-Payload-Blinded`. //! //! A payload's blobs travel with the block: production answers fulu's //! `BlockContents` (cell proofs and blobs beside the block), and publication @@ -25,6 +26,7 @@ use axum::{ response::{IntoResponse, Response}, routing::{get, post}, }; +use ethlambda_blockchain::metrics::SyncStatus; use ethlambda_engine::{ EngineClient, ForkchoiceStateV1, building::{BuiltPayload, PayloadAttributesV3}, @@ -33,9 +35,9 @@ use ethlambda_engine::{ use ethlambda_network_api::RpcToP2PRef; use ethlambda_state_transition::beacon::{ block_production::{ - BlockInputs, Operations, advance_to_slot, assemble_block, empty_sync_aggregate, - pack_attestations, pack_operations, parse_execution_requests, payload_inputs, - verified_sync_aggregate, + BlindedBlockInputs, BlockInputs, Operations, advance_to_slot, assemble_blinded_block, + assemble_block, empty_sync_aggregate, pack_attestations, pack_operations, + parse_execution_requests, payload_inputs, verified_sync_aggregate, }, bls, data_columns::{self, SidecarError}, @@ -49,15 +51,20 @@ use ethlambda_state_transition::beacon::{ use ethlambda_storage::Store; use ethlambda_types::{ beacon::{ + builder::{BlindedBeaconBlock, ValidatorRegistrationV1}, config::Config, containers::{ self, BeaconState, + altair::SyncAggregate, deneb::Blob, - electra::{BeaconBlock, SignedBeaconBlock}, + electra::{Attestation, BeaconBlock, SignedBeaconBlock}, }, fork::ForkName, preset, - primitives::{BlsSignature, Bytes32, ExecutionAddress, KzgProof, Slot}, + primitives::{ + BlsPubkey, BlsSignature, Bytes32, ExecutionAddress, ExecutionBlockHash, KzgProof, Slot, + Uint256, ValidatorIndex, + }, signing::compute_epoch_at_slot, }, primitives::H256, @@ -66,11 +73,22 @@ use libssz::SszEncode as _; use libssz_derive::{SszDecode, SszEncode}; use libssz_types::SszList; use serde::Deserialize; -use tracing::{error, info, warn}; +use std::sync::Arc; +use tracing::{debug, error, info, warn}; +use crate::beacon::bid_selection::wei_u128; +use crate::beacon::builder_client::VersionedBid; +use crate::beacon::builder_flow::{ + Builder, + bid::{self, BidContext}, + breaker, + choice::{self, DEFAULT_BOOST_FACTOR, LocalOffer, Source}, +}; use crate::beacon::graffiti::{self, OwnVersion}; use crate::beacon::{ApiError, BodyEncoding, validator::FeeRecipients, validator::head}; +use crate::metrics::{inc_bid_rejection, inc_builder_fallback, inc_payload_source}; use crate::shared::content::{Encoding, ssz_response, with_consensus_version}; +use crate::shared::optimistic::block_is_optimistic; /// One KZG proof per cell of every blob, fulu's `kzg_proofs` bound. pub(crate) type CellKzgProofs = SszList< @@ -336,6 +354,9 @@ struct ProduceQuery { /// A flag with no value, validated by [`randao_verification`]. #[serde(default)] skip_randao_verification: Option, + /// beacon-APIs' weight on the builder's bid, in percent; absent is 100. + #[serde(default)] + builder_boost_factor: Option, } /// Whether the block's RANDAO reveal is verified, from the request's @@ -395,6 +416,7 @@ async fn get_block( Extension(sync_pool): Extension, Extension(fee_recipients): Extension, Extension(OwnVersion(own_version)): Extension, + builder: Option>>, headers: HeaderMap, ) -> Response { let Ok(slot) = slot.parse::() else { @@ -431,44 +453,107 @@ async fn get_block( &sync_pool, &fee_recipients, &own_version, + builder.as_ref().map(|Extension(builder)| builder.as_ref()), + query.builder_boost_factor.unwrap_or(DEFAULT_BOOST_FACTOR), slot, query.randao_reveal, BlockProcessingOptions { randao }, graffiti, ) .await; - let (contents, payload_value, fork) = match produced { + let (produced, payload_value, fork) = match produced { Ok(produced) => produced, Err(err) => return err.into_response(), }; + let blinded = matches!(produced, Produced::Blinded(_)); let accept = headers.get(header::ACCEPT).and_then(|v| v.to_str().ok()); - let mut response = match Encoding::from_accept(accept) { - Encoding::Ssz => ssz_response(contents.to_ssz()), - Encoding::Json => crate::json_response(serde_json::json!({ + let mut response = match (Encoding::from_accept(accept), produced) { + (Encoding::Ssz, Produced::Full(contents)) => ssz_response(contents.to_ssz()), + (Encoding::Ssz, Produced::Blinded(block)) => ssz_response(block.to_ssz()), + (Encoding::Json, Produced::Full(contents)) => crate::json_response(serde_json::json!({ "version": fork.as_str(), "execution_payload_blinded": false, "execution_payload_value": payload_value, "consensus_block_value": "0", "data": contents, })), + (Encoding::Json, Produced::Blinded(block)) => crate::json_response(serde_json::json!({ + "version": fork.as_str(), + "execution_payload_blinded": true, + "execution_payload_value": payload_value, + "consensus_block_value": "0", + "data": block, + })), }; let headers = response.headers_mut(); headers.insert( "eth-execution-payload-blinded", - HeaderValue::from_static("false"), + HeaderValue::from_static(if blinded { "true" } else { "false" }), ); if let Ok(value) = HeaderValue::from_str(&payload_value) { headers.insert("eth-execution-payload-value", value); } - // Not computed: nothing here reads it, and the builder comparison it - // exists for does not happen on this node. + // Not computed: the builder comparison here weighs the execution + // payloads' values alone, and nothing else on this node reads it. headers.insert("eth-consensus-block-value", HeaderValue::from_static("0")); with_consensus_version(response, fork) } -/// The block for `slot` with its blobs and cell proofs, the payload's value in -/// wei as a decimal string, and the block's fork. +/// What `produceBlockV3` answers with. +enum Produced { + /// The local block, with its blobs and cell proofs. + Full(FuluBlockContents), + /// The blinded block around a builder's bid. + Blinded(BlindedBeaconBlock), +} + +/// The block's consensus contents, packed once against the proposal state +/// and offered to both payloads. +struct Packed { + attestations: Vec, + operations: Operations, + sync_aggregate: SyncAggregate, +} + +impl Packed { + /// Whether dropping the packed contents might save a block that failed to + /// build. + fn droppable(&self) -> bool { + !self.attestations.is_empty() + || !self.operations.is_empty() + || self.sync_aggregate != empty_sync_aggregate() + } +} + +/// The local block and what its payload offers. +struct LocalBlock { + contents: FuluBlockContents, + value: Uint256, + should_override_builder: bool, +} + +/// The blinded block around a validated bid, and what the bid pays. +struct BlindedCandidate { + block: BlindedBeaconBlock, + value: Uint256, +} + +/// What to ask the builder for this slot. +struct BuilderRequest<'a> { + builder: &'a Builder, + pubkey: BlsPubkey, + registration: ValidatorRegistrationV1, + parent_hash: ExecutionBlockHash, +} + +/// The block for `slot`, its payload's value in wei as a decimal string, and +/// the block's fork. +/// +/// With a builder layered, the proposer registered through this node and the +/// circuit breaker quiet, the builder's bid is fetched alongside the local +/// payload. Whichever [`choice::choose`] picks goes into the block. Otherwise +/// the block is built exactly as it is without a builder. #[allow(clippy::too_many_arguments)] async fn produce( store: &Store, @@ -476,11 +561,13 @@ async fn produce( sync_pool: &SharedSyncCommitteePool, fee_recipients: &FeeRecipients, own_version: &ClientVersionV1, + builder: Option<&Builder>, + boost_factor: u64, slot: Slot, randao_reveal: BlsSignature, options: BlockProcessingOptions, graffiti: Bytes32, -) -> Result<(FuluBlockContents, String, ForkName), ApiError> { +) -> Result<(Produced, String, ForkName), ApiError> { let config = store.config(); let (head_root, head_state) = head(store)?; if slot <= head_state.slot() { @@ -492,16 +579,132 @@ async fn produce( let proposer = ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index(&state) .map_err(|_| ApiError::Internal("no proposer for the slot"))?; + let request = + builder.and_then(|builder| builder_request(store, builder, &state, head_root, proposer)); // Asked alongside the build rather than before it, so a healthy execution // client costs the block nothing for its version and a silent one costs it - // at most the version's own timeout. - let (built, el_version) = tokio::join!( + // at most the version's own timeout. The builder's bid is asked alongside + // as well, bounded by its own header timeout. + let (built, el_version, bid) = tokio::join!( build_payload(store, engine, fee_recipients, &state, head_root, proposer), graffiti::execution_client_version(engine, own_version), + fetch_bid(request.as_ref(), slot), ); - let mut built = built?; let graffiti = graffiti::with_client_versions(graffiti, el_version.as_ref(), own_version); + let packed = pack(store, sync_pool, &state, head_root, slot, &config); + let local = built.and_then(|built| { + local_block( + &state, + built, + &packed, + randao_reveal, + graffiti, + &config, + options, + slot, + ) + }); + let blinded = request.as_ref().zip(bid).and_then(|(request, bid)| { + blinded_block( + &state, + &config, + request, + bid, + &packed, + randao_reveal, + graffiti, + options, + ) + }); + + let local_offer = local.as_ref().ok().map(|local| LocalOffer { + value: wei_u128(&local.value), + should_override_builder: local.should_override_builder, + }); + let bid_value = blinded.as_ref().map(|blinded| wei_u128(&blinded.value)); + if choice::choose(local_offer, bid_value, boost_factor) == Some(Source::Builder) + && let Some(blinded) = blinded + { + inc_payload_source(Source::Builder.label()); + let value = decimal(&blinded.value); + info!(%slot, proposer, %value, "Produced a blinded block around the builder's bid"); + return Ok((Produced::Blinded(blinded.block), value, fork)); + } + // The local error surfaces only when the builder did not step in. + let local = local?; + if builder.is_some() { + inc_payload_source(Source::Local.label()); + } + let block = &local.contents.block; + info!( + %slot, + proposer, + attestations = block.body.attestations.len(), + slashings = block.body.proposer_slashings.len() + block.body.attester_slashings.len(), + exits = block.body.voluntary_exits.len(), + bls_changes = block.body.bls_to_execution_changes.len(), + transactions = block.body.execution_payload.transactions.len(), + blobs = block.body.blob_kzg_commitments.len(), + graffiti = %graffiti::display(&block.body.graffiti), + "Produced block" + ); + Ok((Produced::Full(local.contents), decimal(&local.value), fork)) +} + +/// The block's consensus contents for `state`: attestations, operations and +/// the sync aggregate, each checked against `state`. +fn pack( + store: &Store, + sync_pool: &SharedSyncCommitteePool, + state: &BeaconState, + head_root: H256, + slot: Slot, + config: &Config, +) -> Packed { + let candidates = store.attestation_pool().block_candidates(); + let attestations = pack_attestations(state, candidates); + let operation_candidates = { + let pool = store.operation_pool(); + Operations { + proposer_slashings: pool.proposer_slashings(), + attester_slashings: pool.attester_slashings(), + voluntary_exits: pool.voluntary_exits(), + bls_to_execution_changes: pool.bls_to_execution_changes(), + } + }; + let operations = pack_operations(state, operation_candidates, config); + // What the committee signed at the previous slot over this block's parent, + // verified exactly as `process_sync_aggregate` will check it against the + // state the block is built on; a candidate that fails becomes the empty + // aggregate rather than a block the chain would refuse. + let candidate = sync_pool + .lock() + .expect("sync committee pool lock poisoned") + .sync_aggregate(slot - 1, head_root); + let sync_aggregate = candidate.map_or_else(empty_sync_aggregate, |candidate| { + verified_sync_aggregate(state, candidate) + }); + Packed { + attestations, + operations, + sync_aggregate, + } +} + +/// The local block: the execution client's payload around `packed`, with +/// its blobs, cell proofs and value. +#[allow(clippy::too_many_arguments)] +fn local_block( + state: &BeaconState, + mut built: BuiltPayload, + packed: &Packed, + randao_reveal: BlsSignature, + graffiti: Bytes32, + config: &Config, + options: BlockProcessingOptions, + slot: Slot, +) -> Result { let bundle = &built.blobs_bundle; let malformed = bundle.commitments.len() != bundle.blobs.len() || bundle.proofs.len() != bundle.blobs.len() * preset::CELLS_PER_EXT_BLOB; @@ -534,32 +737,7 @@ async fn produce( }; let execution_requests = parse_execution_requests(&built.execution_requests) .map_err(|_| ApiError::Internal("the execution client's request list is malformed"))?; - let payload_value = decimal(&built.block_value); - let candidates = store.attestation_pool().block_candidates(); - let attestations = pack_attestations(&state, candidates); - let operation_candidates = { - let pool = store.operation_pool(); - Operations { - proposer_slashings: pool.proposer_slashings(), - attester_slashings: pool.attester_slashings(), - voluntary_exits: pool.voluntary_exits(), - bls_to_execution_changes: pool.bls_to_execution_changes(), - } - }; - let operations = pack_operations(&state, operation_candidates, &config); - // What the committee signed at the previous slot over this block's parent, - // verified exactly as `process_sync_aggregate` will check it against the - // state the block is built on; a candidate that fails becomes the empty - // aggregate rather than a block the chain would refuse. - let candidate = sync_pool - .lock() - .expect("sync committee pool lock poisoned") - .sync_aggregate(slot - 1, head_root); - let sync_aggregate = candidate.map_or_else(empty_sync_aggregate, |candidate| { - verified_sync_aggregate(&state, candidate) - }); - let has_sync_aggregate = sync_aggregate != empty_sync_aggregate(); let inputs = |attestations, operations, sync_aggregate| BlockInputs { sync_aggregate, randao_reveal, @@ -570,49 +748,171 @@ async fn produce( blob_kzg_commitments: built.blobs_bundle.commitments.clone(), execution_requests: execution_requests.clone(), }; - let attestation_count = attestations.len(); - let has_operations = !operations.is_empty(); let block = match assemble_block( - &state, - inputs(attestations, operations, sync_aggregate), - &config, + state, + inputs( + packed.attestations.clone(), + packed.operations.clone(), + packed.sync_aggregate.clone(), + ), + config, options, ) { Ok(block) => block, // `pack_attestations` and `pack_operations` check every candidate - // against this state, and the sync aggregate was verified above, so this - // should not happen; but a block without them still earns the proposal, - // and one that fails to build earns nothing. - Err(err) if attestation_count > 0 || has_operations || has_sync_aggregate => { + // against this state, and the sync aggregate is verified against it + // too, so this should not happen; but a block without them still earns + // the proposal, and one that fails to build earns nothing. + Err(err) if packed.droppable() => { warn!(%slot, %err, "Block with attestations, operations or a sync aggregate failed to build; retrying without"); assemble_block( - &state, + state, inputs(Vec::new(), Operations::default(), empty_sync_aggregate()), - &config, + config, options, ) .map_err(|_| ApiError::Internal("the block failed to build"))? } Err(_) => return Err(ApiError::Internal("the block failed to build")), }; - info!( - %slot, - proposer, - attestations = block.body.attestations.len(), - slashings = block.body.proposer_slashings.len() + block.body.attester_slashings.len(), - exits = block.body.voluntary_exits.len(), - bls_changes = block.body.bls_to_execution_changes.len(), - transactions = block.body.execution_payload.transactions.len(), - blobs = block.body.blob_kzg_commitments.len(), - graffiti = %graffiti::display(&block.body.graffiti), - "Produced block" - ); - let contents = FuluBlockContents { - block, - kzg_proofs, - blobs, + Ok(LocalBlock { + contents: FuluBlockContents { + block, + kzg_proofs, + blobs, + }, + value: built.block_value, + should_override_builder: built.should_override_builder, + }) +} + +/// What to ask the builder for, if anything. +/// +/// The builder is asked only when the proposer registered through this node +/// and the circuit breaker is quiet. A validator client that does not use +/// builders sends no registrations, so its validators never get here. +/// `get_block` has already refused every fork but fulu. +fn builder_request<'a>( + store: &Store, + builder: &'a Builder, + state: &BeaconState, + head_root: H256, + proposer: ValidatorIndex, +) -> Option> { + let BeaconState::Fulu(fulu) = state else { + return None; }; - Ok((contents, payload_value, fork)) + let pubkey = state.validator(proposer).ok()?.pubkey; + let registration = builder + .registrations + .get(&pubkey, compute_epoch_at_slot(state.slot()))?; + let optimistic = block_is_optimistic(store, head_root); + let syncing = builder.sync_status.get() == SyncStatus::Syncing; + if let Err(trip) = breaker::check(state, &builder.fallback, optimistic, syncing) { + inc_builder_fallback(trip.label()); + info!( + slot = state.slot(), + reason = trip.label(), + "The circuit breaker keeps the builder out; building locally" + ); + return None; + } + Some(BuilderRequest { + builder, + pubkey, + registration, + parent_hash: fulu.latest_execution_payload_header.block_hash, + }) +} + +/// The builder's bid, or `None` when it was not asked, had none, or failed. +async fn fetch_bid(request: Option<&BuilderRequest<'_>>, slot: Slot) -> Option { + let request = request?; + request + .builder + .client + .get_header(slot, request.parent_hash, &request.pubkey) + .await + .inspect_err(|err| warn!(%slot, %err, "The builder gave no bid; building locally")) + .ok() + .flatten() +} + +/// The blinded block around `bid`, if the bid is acceptable and passes the +/// state transition. +#[allow(clippy::too_many_arguments)] +fn blinded_block( + state: &BeaconState, + config: &Config, + request: &BuilderRequest, + bid: VersionedBid, + packed: &Packed, + randao_reveal: BlsSignature, + graffiti: Bytes32, + options: BlockProcessingOptions, +) -> Option { + let slot = state.slot(); + let BeaconState::Fulu(fulu) = state else { + return None; + }; + let inputs = payload_inputs(state, config).ok()?; + let context = BidContext { + genesis_fork_version: config.genesis_fork_version, + inputs: &inputs, + parent_gas_limit: fulu.latest_execution_payload_header.gas_limit, + target_gas_limit: request.registration.gas_limit, + max_blobs: config.max_blobs_per_block(compute_epoch_at_slot(slot)), + }; + if let Err(rejection) = bid::validate(&bid.bid, &context) { + inc_bid_rejection(rejection.label()); + warn!(%slot, reason = rejection.label(), "Refusing the builder's bid"); + return None; + } + let message = bid.bid.message; + if message.header.fee_recipient != request.registration.fee_recipient { + // Expected: relays pay the proposer with a transaction in the payload + // and name the builder as its fee recipient (docs/spec_deviations.md). + debug!(%slot, "The bid's fee recipient is not the registered one"); + } + let assemble = |attestations, operations, sync_aggregate| { + assemble_blinded_block( + state, + BlindedBlockInputs { + randao_reveal, + graffiti, + attestations, + operations, + execution_payload_header: message.header.clone(), + blob_kzg_commitments: message.blob_kzg_commitments.to_vec(), + execution_requests: message.execution_requests.clone(), + sync_aggregate, + }, + config, + options, + ) + }; + let assembled = match assemble( + packed.attestations.clone(), + packed.operations.clone(), + packed.sync_aggregate.clone(), + ) { + Err(err) if packed.droppable() => { + warn!(%slot, %err, "Blinded block with attestations, operations or a sync aggregate failed to build; retrying without"); + assemble(Vec::new(), Operations::default(), empty_sync_aggregate()) + } + other => other, + }; + match assembled { + Ok(block) => Some(BlindedCandidate { + block, + value: message.value, + }), + Err(err) => { + inc_bid_rejection("state_transition"); + warn!(%slot, %err, "The builder's bid fails the state transition"); + None + } + } } /// Ask the execution client to build on the head for `state`'s slot, then @@ -1025,4 +1325,268 @@ mod tests { assert_eq!(status, StatusCode::BAD_REQUEST); assert!(rejected.blocks.lock().unwrap().is_empty()); } + + use crate::beacon::builder_client::mock_relay::MockRelay; + use crate::beacon::builder_client::{BuilderClient, VersionedBid}; + use crate::beacon::builder_flow::breaker::FallbackThresholds; + use ethlambda_blockchain::SyncStatusController; + use ethlambda_types::beacon::builder::{BuilderBid, ValidatorRegistrationV1}; + use ethlambda_types::beacon::containers::capella; + use ethlambda_types::beacon::containers::deneb::{ExecutionPayloadHeader, KzgCommitments}; + use ethlambda_types::beacon::containers::electra::{self, ExecutionRequests}; + use ethlambda_types::beacon::primitives::HashTreeRoot as _; + use libssz::SszDecode as _; + use std::time::Duration; + + const ETH: u64 = 1_000_000_000_000_000_000; + /// The registered gas limit the test proposer asks for. + const TARGET_GAS_LIMIT: u64 = 60_000_000; + + /// Thresholds that never trip, so a test state's history does not decide + /// whether the builder is asked. + const NEVER_TRIP: FallbackThresholds = FallbackThresholds { + skips: u64::MAX, + skips_per_epoch: u64::MAX, + epochs_since_finalization: u64::MAX, + }; + + /// A builder world for the next slot on `fulu_store()`: a mock relay, a + /// `Builder` pointing at it with the proposer registered, and the + /// advanced proposal state. + struct BuilderWorld { + store: Store, + slot: Slot, + state: BeaconState, + relay: MockRelay, + builder: Arc, + } + + async fn builder_world(registered: bool, sync: SyncStatus) -> BuilderWorld { + let (store, head_slot) = fulu_store(); + let slot = head_slot + 1; + let config = store.config(); + let (_, head_state) = super::super::validator::head(&store).unwrap(); + let state = advance_to_slot(&head_state, slot, &config).unwrap(); + let proposer = get_beacon_proposer_index(&state).unwrap(); + let pubkey = state.validator(proposer).unwrap().pubkey; + let relay = MockRelay::start(config.genesis_fork_version).await; + let client = + BuilderClient::new(relay.url.clone(), Duration::from_secs(1), "ethlambda/test") + .unwrap(); + let builder = Arc::new(Builder::new( + client, + NEVER_TRIP, + SyncStatusController::new(sync), + )); + if registered { + let registration = ValidatorRegistrationV1 { + fee_recipient: ExecutionAddress::repeat_byte(0x11), + gas_limit: TARGET_GAS_LIMIT, + timestamp: 1, + pubkey, + }; + builder + .registrations + .record([®istration], compute_epoch_at_slot(slot)); + } + BuilderWorld { + store, + slot, + state, + relay, + builder, + } + } + + impl BuilderWorld { + /// A bid of `value` wei that passes every check for this slot, with + /// `edit` applied before the relay signs it. + fn offer(&self, value: u64, edit: impl FnOnce(&mut BuilderBid)) { + let config = self.store.config(); + let inputs = payload_inputs(&self.state, &config).unwrap(); + let BeaconState::Fulu(fulu) = &self.state else { + unreachable!("built as fulu") + }; + let parent_gas_limit = fulu.latest_execution_payload_header.gas_limit; + let step = (parent_gas_limit / 1024).saturating_sub(1); + let withdrawals: capella::Withdrawals = inputs.withdrawals.clone().try_into().unwrap(); + let mut header = + ExecutionPayloadHeader::from(&electra::BeaconBlockBody::empty().execution_payload); + header.parent_hash = inputs.parent_hash; + header.prev_randao = inputs.prev_randao; + header.timestamp = inputs.timestamp; + header.withdrawals_root = withdrawals.hash_tree_root(); + header.gas_limit = TARGET_GAS_LIMIT.clamp( + parent_gas_limit - step, + parent_gas_limit.saturating_add(step), + ); + header.block_hash = ExecutionBlockHash::repeat_byte(0xbb); + let mut message = BuilderBid { + header, + blob_kzg_commitments: KzgCommitments::default(), + execution_requests: ExecutionRequests::default(), + value: Uint256::from(value), + pubkey: self.relay.pubkey(), + }; + edit(&mut message); + let bid = self.relay.sign_bid(message); + self.relay.offer(VersionedBid { + fork: ForkName::Fulu, + bid, + }); + } + + fn app(&self, engine: Option) -> Router { + produce_app(self.store.clone(), engine).layer(Extension(self.builder.clone())) + } + + fn uri(&self, query: &str) -> String { + format!( + "{}{query}", + produce_uri(self.slot, &infinity_reveal(), "&skip_randao_verification") + ) + } + } + + async fn produce_json( + world: &BuilderWorld, + engine_value: &'static str, + override_builder: bool, + query: &str, + ) -> (StatusCode, serde_json::Value) { + let engine = super::super::validator_client_tests::fake_execution_client_paying( + engine_value, + override_builder, + ) + .await; + let request = Request::get(world.uri(query)).body(Body::empty()).unwrap(); + respond(world.app(Some(engine)), request).await + } + + fn served_blinded(json: &serde_json::Value) -> bool { + json["execution_payload_blinded"] == serde_json::json!(true) + } + + #[tokio::test] + async fn a_bid_worth_more_than_the_local_payload_is_served_blinded() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |_| {}); + let (status, json) = produce_json(&world, "0x2a", false, "").await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert!(served_blinded(&json), "{json}"); + assert_eq!( + json["execution_payload_value"], + serde_json::json!(ETH.to_string()) + ); + assert_eq!( + json["data"]["slot"], + serde_json::json!(world.slot.to_string()) + ); + assert_eq!( + json["data"]["body"]["execution_payload_header"]["block_hash"], + serde_json::json!(format!("0x{}", "bb".repeat(32))) + ); + } + + #[tokio::test] + async fn a_tie_keeps_the_local_payload() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |_| {}); + // 1 ETH in hex wei. + let (status, json) = produce_json(&world, "0xde0b6b3a7640000", false, "").await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert!(!served_blinded(&json), "{json}"); + } + + #[tokio::test] + async fn a_boost_factor_of_zero_keeps_the_local_payload() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |_| {}); + let (_, json) = produce_json(&world, "0x2a", false, "&builder_boost_factor=0").await; + assert!(!served_blinded(&json), "{json}"); + } + + #[tokio::test] + async fn the_maximum_boost_factor_takes_the_bid() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(1_000_000_000, |_| {}); + let query = format!("&builder_boost_factor={}", u64::MAX); + let (_, json) = produce_json(&world, "0xde0b6b3a7640000", false, &query).await; + assert!(served_blinded(&json), "{json}"); + } + + #[tokio::test] + async fn the_execution_clients_override_keeps_the_local_payload() { + let world = builder_world(true, SyncStatus::Synced).await; + // 10 ETH: the largest round value a u64 of wei holds. + world.offer(10 * ETH, |_| {}); + let (_, json) = produce_json(&world, "0x2a", true, "").await; + assert!(!served_blinded(&json), "{json}"); + } + + #[tokio::test] + async fn an_unregistered_proposer_never_reaches_the_builder() { + let world = builder_world(false, SyncStatus::Synced).await; + world.offer(ETH, |_| {}); + let (_, json) = produce_json(&world, "0x2a", false, "").await; + assert!(!served_blinded(&json), "{json}"); + assert!(world.relay.header_requests().is_empty()); + } + + #[tokio::test] + async fn a_syncing_node_keeps_the_builder_out() { + let world = builder_world(true, SyncStatus::Syncing).await; + world.offer(ETH, |_| {}); + let (_, json) = produce_json(&world, "0x2a", false, "").await; + assert!(!served_blinded(&json), "{json}"); + assert!(world.relay.header_requests().is_empty()); + } + + #[tokio::test] + async fn a_bid_on_the_wrong_parent_is_dropped() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |bid| { + bid.header.parent_hash = ExecutionBlockHash::repeat_byte(9) + }); + let (status, json) = produce_json(&world, "0x2a", false, "").await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert!(!served_blinded(&json), "{json}"); + } + + #[tokio::test] + async fn without_a_local_payload_the_bid_is_served() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |_| {}); + let dead_engine = EngineClient::new( + "http://127.0.0.1:1".to_owned(), + ethlambda_engine::JwtSecret::new([0x0f; 32]), + ) + .unwrap(); + let request = Request::get(world.uri("")).body(Body::empty()).unwrap(); + let (status, json) = respond(world.app(Some(dead_engine)), request).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert!(served_blinded(&json), "{json}"); + } + + #[tokio::test] + async fn a_blinded_block_is_served_in_ssz_with_its_headers() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |_| {}); + let engine = + super::super::validator_client_tests::fake_execution_client_paying("0x2a", false).await; + let request = Request::get(world.uri("")) + .header("accept", "application/octet-stream") + .body(Body::empty()) + .unwrap(); + let response = world.app(Some(engine)).oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["eth-execution-payload-blinded"], "true"); + assert_eq!( + response.headers()["eth-execution-payload-value"], + ETH.to_string().as_str() + ); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let block = BlindedBeaconBlock::from_ssz_bytes(&body).unwrap(); + assert_eq!(block.slot, world.slot); + } } diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 693450ebf..654fb6d5d 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -420,18 +420,29 @@ fn valid_blob(seed: u8) -> Vec { /// The fulu bundle carries `blob_count` valid blobs with their commitments and /// cell proofs (flattened blob by blob, as `BlobsBundleV2` does). pub(super) async fn fake_execution_client(blob_count: usize) -> ethlambda_engine::EngineClient { - fake_execution_client_with(blob_count, None).await + fake_execution_client_with(blob_count, None, "0x2a", false).await +} + +/// [`fake_execution_client`] with no blobs, whose fulu payload pays +/// `block_value` (hex wei) and carries `should_override_builder`. +pub(super) async fn fake_execution_client_paying( + block_value: &'static str, + should_override_builder: bool, +) -> ethlambda_engine::EngineClient { + fake_execution_client_with(0, None, block_value, should_override_builder).await } /// [`fake_execution_client`] whose gloas payloads carry `blob`, with its real /// commitment and cell proofs, in the blobs bundle. async fn fake_execution_client_with_blob(blob: Option>) -> ethlambda_engine::EngineClient { - fake_execution_client_with(0, blob).await + fake_execution_client_with(0, blob, "0x2a", false).await } async fn fake_execution_client_with( blob_count: usize, blob: Option>, + block_value: &'static str, + should_override_builder: bool, ) -> ethlambda_engine::EngineClient { use axum::{Json, routing::post}; use ethlambda_state_transition::beacon::kzg; @@ -513,9 +524,9 @@ async fn fake_execution_client_with( v5_payload["blobGasUsed"] = blob_gas_used.clone().into(); serde_json::json!({ "executionPayload": v5_payload, - "blockValue": "0x2a", + "blockValue": block_value, "blobsBundle": { "commitments": commitments, "proofs": proofs, "blobs": blobs }, - "shouldOverrideBuilder": false, + "shouldOverrideBuilder": should_override_builder, "executionRequests": [], }) } diff --git a/crates/net/rpc/src/metrics.rs b/crates/net/rpc/src/metrics.rs index 14baad83e..913f7d7d8 100644 --- a/crates/net/rpc/src/metrics.rs +++ b/crates/net/rpc/src/metrics.rs @@ -84,3 +84,51 @@ pub(crate) fn inc_builder_registrations(outcome: &str, count: u64) { .with_label_values(&[outcome]) .inc_by(count); } + +/// Count builder bids refused before selection, by reason. +pub(crate) fn inc_bid_rejection(reason: &str) { + static LEAN_BEACON_BUILDER_BID_REJECTIONS_TOTAL: LazyLock = + LazyLock::new(|| { + register_int_counter_vec!( + "lean_beacon_builder_bid_rejections_total", + "Builder bids refused before selection, by reason", + &["reason"] + ) + .unwrap() + }); + LEAN_BEACON_BUILDER_BID_REJECTIONS_TOTAL + .with_label_values(&[reason]) + .inc(); +} + +/// Count blocks produced with a builder configured, by where the payload +/// came from. +pub(crate) fn inc_payload_source(source: &str) { + static LEAN_BEACON_BUILDER_PAYLOAD_SOURCE_TOTAL: LazyLock = + LazyLock::new(|| { + register_int_counter_vec!( + "lean_beacon_builder_payload_source_total", + "Blocks produced with a builder configured, by where the payload came from", + &["source"] + ) + .unwrap() + }); + LEAN_BEACON_BUILDER_PAYLOAD_SOURCE_TOTAL + .with_label_values(&[source]) + .inc(); +} + +/// Count proposals the circuit breaker kept from the builder, by reason. +pub(crate) fn inc_builder_fallback(reason: &str) { + static LEAN_BEACON_BUILDER_FALLBACK_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter_vec!( + "lean_beacon_builder_fallback_total", + "Proposals the circuit breaker kept from the builder, by reason", + &["reason"] + ) + .unwrap() + }); + LEAN_BEACON_BUILDER_FALLBACK_TOTAL + .with_label_values(&[reason]) + .inc(); +} From 5cd832cae50a9275c8051a1385e42a1f04a051e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:47:13 -0300 Subject: [PATCH 08/20] feat(rpc): hand signed blinded blocks to the builder publishBlindedBlockV2 checks a blinded block the way publishBlock checks a full one, the signature through a root-based check the two now share, then submits it on its own task so a validator client hanging up cannot cancel the reveal. Both endpoints claim the slot first: a second, different block for a slot is refused, since this node handing out two would get its validator slashed. --- .../state_transition/src/beacon/stf/mod.rs | 27 ++- crates/net/rpc/src/beacon/blinded_blocks.rs | 182 ++++++++++++++++++ crates/net/rpc/src/beacon/mod.rs | 1 + crates/net/rpc/src/beacon/proposal.rs | 162 +++++++++++++++- 4 files changed, 363 insertions(+), 9 deletions(-) create mode 100644 crates/net/rpc/src/beacon/blinded_blocks.rs diff --git a/crates/blockchain/state_transition/src/beacon/stf/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/mod.rs index 3dba6eef8..17672afd1 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/mod.rs @@ -71,7 +71,7 @@ use crate::beacon::helpers::accessors::{ActiveBalanceCache, CommitteeCache, get_ use crate::beacon::helpers::misc::compute_signing_root; use crate::beacon::lean_state_unreachable; use crate::beacon::preset; -use crate::beacon::primitives::{HashTreeRoot as _, Slot}; +use crate::beacon::primitives::{BlsSignature, HashTreeRoot as _, Root, Slot, ValidatorIndex}; use crate::beacon::{bls, config::Config, constants}; /// What the execution layer would answer for a block's payload. @@ -212,12 +212,31 @@ pub fn verify_block_signature( state: &BeaconState, signed_block: &containers::SignedBeaconBlock, ) -> bool { - let Ok(proposer) = state.validator(signed_block.proposer_index()) else { + verify_proposer_signature( + state, + signed_block.proposer_index(), + signed_block.message_hash_tree_root(), + &signed_block.signature(), + ) +} + +/// Whether `signature` is `proposer_index`'s over the block whose root is +/// `block_root`, under `state`'s proposer domain. +/// +/// A blinded block has the root of its full block, so one check serves +/// both. +pub fn verify_proposer_signature( + state: &BeaconState, + proposer_index: ValidatorIndex, + block_root: Root, + signature: &BlsSignature, +) -> bool { + let Ok(proposer) = state.validator(proposer_index) else { return false; }; let domain = get_domain(state, constants::DOMAIN_BEACON_PROPOSER, None); - let signing_root = compute_signing_root(signed_block.message_hash_tree_root(), domain); - bls::verify(&proposer.pubkey, signing_root, &signed_block.signature()) + let signing_root = compute_signing_root(block_root, domain); + bls::verify(&proposer.pubkey, signing_root, signature) } /// Advances the state to `slot`, running epoch processing at each boundary and diff --git a/crates/net/rpc/src/beacon/blinded_blocks.rs b/crates/net/rpc/src/beacon/blinded_blocks.rs new file mode 100644 index 000000000..82f73f733 --- /dev/null +++ b/crates/net/rpc/src/beacon/blinded_blocks.rs @@ -0,0 +1,182 @@ +//! `POST /eth/v2/beacon/blinded_blocks` (fulu), and the guard both publish +//! endpoints share against signing two blocks for one slot. +//! +//! A blinded block goes to the builder, which publishes the full block and +//! its data columns itself (`submitBlindedBlockV2`). The block then comes +//! back over gossip and imports like any other. Nothing is gossiped or +//! imported here. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use axum::Extension; +use axum::body::Bytes; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Response}; +use ethlambda_state_transition::beacon::block_production::advance_to_slot; +use ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index; +use ethlambda_state_transition::beacon::stf::verify_proposer_signature; +use ethlambda_storage::Store; +use ethlambda_types::beacon::builder::SignedBlindedBeaconBlock; +use ethlambda_types::beacon::fork::ForkName; +use ethlambda_types::beacon::primitives::{HashTreeRoot as _, Root, Slot}; +use ethlambda_types::beacon::signing::compute_start_slot_at_epoch; +use tracing::{error, info, warn}; + +use super::builder_client::BuilderError; +use super::builder_flow::Builder; +use super::proposal::require_fulu_slot; +use super::{ApiError, BodyEncoding}; + +/// The block root this node published or handed to the builder for each +/// slot, so a second, different block for a slot is refused instead of +/// broadcast. Pruned below the finalized slot. +#[derive(Debug, Clone, Default)] +pub(crate) struct PublishedProposals(Arc>>); + +impl PublishedProposals { + /// Claims `slot` for `root`, or refuses with the root already claimed. + /// The same root again is a retry, not an equivocation. + pub(crate) fn claim(&self, slot: Slot, root: Root, finalized_slot: Slot) -> Result<(), Root> { + let mut published = self.0.lock().expect("published proposals lock poisoned"); + *published = published.split_off(&finalized_slot); + match published.get(&slot) { + Some(&claimed) if claimed != root => Err(claimed), + Some(_) => Ok(()), + None => { + published.insert(slot, root); + Ok(()) + } + } + } +} + +/// The finalized checkpoint's start slot, below which no claim matters. +pub(crate) fn finalized_slot(store: &Store) -> Slot { + compute_start_slot_at_epoch(store.beacon_finalized_checkpoint().epoch) +} + +/// Checked like `post_block`'s fulu path, before anything leaves this node: +/// the parent is held, the block is after it, its proposer is the one the +/// parent's state names for the slot, and the proposer's signature verifies. +/// Then the slot is claimed, and the block goes to the builder. +/// +/// The submission runs on its own task. Once the proposer has signed it is +/// committed, so a validator client that hangs up must not cancel the +/// reveal. The answer is the builder's: 202 becomes 200 and a 400 is passed +/// through. Anything else is a 500, with no local fallback, since a second +/// block would be an equivocation. +pub(crate) async fn post_blinded_block( + State(store): State, + builder: Option>>, + Extension(published): Extension, + headers: HeaderMap, + body: Bytes, +) -> Response { + let Some(Extension(builder)) = builder else { + return ApiError::Internal("no builder is configured (--builder)").into_response(); + }; + let fork = headers + .get("eth-consensus-version") + .and_then(|value| value.to_str().ok()) + .and_then(ForkName::parse); + if fork != Some(ForkName::Fulu) { + return ApiError::BadRequest("Eth-Consensus-Version must be fulu").into_response(); + } + let encoding = match BodyEncoding::from_headers(&headers) { + Ok(encoding) => encoding, + Err(err) => return err.into_response(), + }; + let Some(signed) = encoding.decode::(&body) else { + return ApiError::BadRequest("the body is not a fulu SignedBlindedBeaconBlock") + .into_response(); + }; + let slot = signed.message.slot; + if let Err(err) = require_fulu_slot( + &store.config(), + slot, + "blinded blocks are accepted for fulu slots only", + ) { + return err.into_response(); + } + let Some(parent_state) = store.get_state(&signed.message.parent_root).ok().flatten() else { + return ApiError::BadRequest("the block's parent is not known to this node") + .into_response(); + }; + let config = store.config(); + let checked = tokio::task::spawn_blocking(move || { + if slot <= parent_state.slot() { + return Err("the block is not after its parent"); + } + let state = advance_to_slot(&parent_state, slot, &config) + .map_err(|_| "advancing the parent state failed")?; + let proposer = get_beacon_proposer_index(&state).map_err(|_| "no proposer for the slot")?; + if signed.message.proposer_index != proposer { + return Err("wrong proposer for the slot"); + } + let root = signed.message.hash_tree_root(); + if !verify_proposer_signature(&state, proposer, root, &signed.signature) { + return Err("invalid block signature"); + } + Ok((signed, root)) + }) + .await; + let (signed, root) = match checked { + Ok(Ok(checked)) => checked, + Ok(Err(message)) => return ApiError::BadRequest(message).into_response(), + Err(err) => { + error!(%err, "Checking a published blinded block failed"); + return ApiError::Internal("checking the blinded block failed").into_response(); + } + }; + if published.claim(slot, root, finalized_slot(&store)).is_err() { + warn!(slot, "Refusing a second, different block for the slot"); + return ApiError::BadRequest("this node already published a different block for the slot") + .into_response(); + } + + let client = builder.client.clone(); + let submitted = + tokio::spawn(async move { client.submit_blinded_block(&signed, ForkName::Fulu).await }) + .await; + match submitted { + Ok(Ok(())) => { + info!(slot, "The builder took the blinded block"); + StatusCode::OK.into_response() + } + Ok(Err(BuilderError::BadRequest(message))) => { + warn!(slot, %message, "The builder refused the blinded block"); + ApiError::BadRequestDetail(message).into_response() + } + Ok(Err(err)) => { + warn!(slot, %err, "The builder did not take the blinded block"); + ApiError::Internal("the builder did not take the block").into_response() + } + Err(err) => { + error!(%err, "Submitting the blinded block failed"); + ApiError::Internal("submitting the block failed").into_response() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_slot_is_claimed_once_and_a_retry_of_the_same_root_passes() { + let published = PublishedProposals::default(); + let first = Root::repeat_byte(1); + assert_eq!(published.claim(10, first, 0), Ok(())); + assert_eq!(published.claim(10, first, 0), Ok(())); + assert_eq!(published.claim(10, Root::repeat_byte(2), 0), Err(first)); + } + + #[test] + fn claims_below_the_finalized_slot_are_forgotten() { + let published = PublishedProposals::default(); + published.claim(10, Root::repeat_byte(1), 0).unwrap(); + assert_eq!(published.claim(10, Root::repeat_byte(2), 11), Ok(())); + } +} diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index ad509377a..b1f9837f7 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -17,6 +17,7 @@ use crate::shared::block_id::IdError; pub(crate) mod bid_selection; pub(crate) mod bids; +pub(crate) mod blinded_blocks; pub(crate) mod blocks; pub(crate) mod builder_client; pub(crate) mod builder_config; diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index a582394a1..315c7dc57 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -62,8 +62,8 @@ use ethlambda_types::{ fork::ForkName, preset, primitives::{ - BlsPubkey, BlsSignature, Bytes32, ExecutionAddress, ExecutionBlockHash, KzgProof, Slot, - Uint256, ValidatorIndex, + BlsPubkey, BlsSignature, Bytes32, ExecutionAddress, ExecutionBlockHash, + HashTreeRoot as _, KzgProof, Slot, Uint256, ValidatorIndex, }, signing::compute_epoch_at_slot, }, @@ -77,6 +77,7 @@ use std::sync::Arc; use tracing::{debug, error, info, warn}; use crate::beacon::bid_selection::wei_u128; +use crate::beacon::blinded_blocks::{self, PublishedProposals, finalized_slot}; use crate::beacon::builder_client::VersionedBid; use crate::beacon::builder_flow::{ Builder, @@ -126,6 +127,11 @@ pub(crate) fn routes() -> Router { "/eth/v2/beacon/blocks", post(post_block).layer(DefaultBodyLimit::max(super::MAX_PUBLISH_BODY_BYTES)), ) + .route( + "/eth/v2/beacon/blinded_blocks", + post(blinded_blocks::post_blinded_block), + ) + .layer(Extension(PublishedProposals::default())) } /// `POST /eth/v2/beacon/blocks`, `SignedBlockContents` (fulu) or a bare @@ -144,6 +150,7 @@ pub(crate) fn routes() -> Router { async fn post_block( State(store): State, Extension(p2p): Extension, + Extension(published): Extension, headers: HeaderMap, body: Bytes, ) -> Response { @@ -159,7 +166,7 @@ async fn post_block( Err(err) => return err.into_response(), }; if fork == Some(ForkName::Gloas) { - return post_gloas_block(&store, &p2p, encoding, &body).await; + return post_gloas_block(&store, &p2p, &published, encoding, &body).await; } let Some(contents) = encoding.decode::(&body) else { return ApiError::BadRequest("the body is not fulu SignedBlockContents").into_response(); @@ -245,6 +252,18 @@ async fn post_block( return ApiError::Internal("building the data column sidecars failed").into_response(); } }; + let root = signed_block.message.hash_tree_root(); + if published + .claim(signed_block.message.slot, root, finalized_slot(&store)) + .is_err() + { + warn!( + slot = signed_block.message.slot, + "Refusing a second, different block for the slot" + ); + return ApiError::BadRequest("this node already published a different block for the slot") + .into_response(); + } let block = containers::SignedBeaconBlock::Fulu(signed_block); let sidecars = sidecars .into_iter() @@ -268,6 +287,7 @@ async fn post_block( async fn post_gloas_block( store: &Store, p2p: &RpcToP2PRef, + published: &PublishedProposals, encoding: BodyEncoding, body: &[u8], ) -> Response { @@ -318,6 +338,21 @@ async fn post_gloas_block( Ok(Err(err)) => return err.into_response(), Err(_) => return ApiError::Internal("verifying the block failed").into_response(), } + if published + .claim( + block.slot(), + block.message_hash_tree_root(), + finalized_slot(store), + ) + .is_err() + { + warn!( + slot = block.slot(), + "Refusing a second, different block for the slot" + ); + return ApiError::BadRequest("this node already published a different block for the slot") + .into_response(); + } // A gloas block carries no blobs of its own: its data columns travel with // the execution payload envelope, published separately, so there are no // sidecars to gossip with the block. @@ -393,7 +428,11 @@ pub(super) fn randao_verification( /// advances a state to it: advancing a fulu head into a gloas slot would run /// `upgrade_to_gloas` only to refuse the result. Every other fork is refused /// too, since the containers here are fulu's. -fn require_fulu_slot(config: &Config, slot: Slot, message: &'static str) -> Result<(), ApiError> { +pub(super) fn require_fulu_slot( + config: &Config, + slot: Slot, + message: &'static str, +) -> Result<(), ApiError> { match config.fork_at_epoch(compute_epoch_at_slot(slot)) { ForkName::Fulu => Ok(()), ForkName::Phase0 @@ -1334,7 +1373,6 @@ mod tests { use ethlambda_types::beacon::containers::capella; use ethlambda_types::beacon::containers::deneb::{ExecutionPayloadHeader, KzgCommitments}; use ethlambda_types::beacon::containers::electra::{self, ExecutionRequests}; - use ethlambda_types::beacon::primitives::HashTreeRoot as _; use libssz::SszDecode as _; use std::time::Duration; @@ -1589,4 +1627,118 @@ mod tests { let block = BlindedBeaconBlock::from_ssz_bytes(&body).unwrap(); assert_eq!(block.slot, world.slot); } + + use ethlambda_state_transition::beacon::helpers::accessors::get_domain; + use ethlambda_state_transition::beacon::helpers::misc::compute_signing_root; + use ethlambda_state_transition::beacon::helpers::test_state::sign_for; + use ethlambda_types::beacon::builder::SignedBlindedBeaconBlock; + use ethlambda_types::beacon::constants::DOMAIN_BEACON_PROPOSER; + use ethlambda_types::beacon::primitives::Root; + + /// A blinded block for `world`'s slot on its head, signed by the slot's + /// proposer (or by `signer`, to forge it), with `graffiti` to make + /// distinct blocks. + fn signed_blinded( + world: &BuilderWorld, + graffiti: u8, + signer: Option, + ) -> SignedBlindedBeaconBlock { + let (head_root, _) = super::super::validator::head(&world.store).unwrap(); + let proposer = get_beacon_proposer_index(&world.state).unwrap(); + let mut body = electra::BeaconBlockBody::empty(); + body.graffiti = Bytes32::repeat_byte(graffiti); + let block = electra::BeaconBlock { + slot: world.slot, + proposer_index: proposer, + parent_root: head_root, + state_root: Root::ZERO, + body, + }; + let message = BlindedBeaconBlock::from(block); + let domain = get_domain(&world.state, DOMAIN_BEACON_PROPOSER, None); + let root = compute_signing_root(message.hash_tree_root(), domain); + let signature = sign_for(signer.unwrap_or(proposer as usize), root); + SignedBlindedBeaconBlock { message, signature } + } + + async fn post_blinded( + app: Router, + block: &SignedBlindedBeaconBlock, + ) -> (StatusCode, serde_json::Value) { + let request = Request::post("/eth/v2/beacon/blinded_blocks") + .header("content-type", "application/octet-stream") + .header("eth-consensus-version", "fulu") + .body(Body::from(block.to_ssz())) + .unwrap(); + respond(app, request).await + } + + fn publish_app(world: &BuilderWorld) -> Router { + let network: RpcToP2PRef = Arc::new(RecordingNetwork::default()); + routes() + .with_state(world.store.clone()) + .layer(Extension(network)) + .layer(Extension(world.builder.clone())) + } + + #[tokio::test] + async fn a_signed_blinded_block_goes_to_the_builder() { + let world = builder_world(true, SyncStatus::Synced).await; + let block = signed_blinded(&world, 1, None); + let (status, json) = post_blinded(publish_app(&world), &block).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(world.relay.blinded_blocks(), [block]); + } + + #[tokio::test] + async fn a_forged_blinded_block_never_reaches_the_builder() { + let world = builder_world(true, SyncStatus::Synced).await; + let proposer = get_beacon_proposer_index(&world.state).unwrap() as usize; + let forged = signed_blinded(&world, 1, Some(proposer + 1)); + let (status, _) = post_blinded(publish_app(&world), &forged).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(world.relay.blinded_blocks().is_empty()); + } + + #[tokio::test] + async fn a_second_different_block_for_the_slot_is_refused() { + let world = builder_world(true, SyncStatus::Synced).await; + let app = publish_app(&world); + let first = signed_blinded(&world, 1, None); + let second = signed_blinded(&world, 2, None); + assert_eq!(post_blinded(app.clone(), &first).await.0, StatusCode::OK); + assert_eq!(post_blinded(app.clone(), &first).await.0, StatusCode::OK); + let (status, json) = post_blinded(app, &second).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(world.relay.blinded_blocks(), [first.clone(), first]); + } + + #[tokio::test] + async fn the_builders_answer_decides_the_status() { + for (answer, expected) in [ + (StatusCode::BAD_REQUEST, StatusCode::BAD_REQUEST), + ( + StatusCode::INTERNAL_SERVER_ERROR, + StatusCode::INTERNAL_SERVER_ERROR, + ), + ] { + let world = builder_world(true, SyncStatus::Synced).await; + world.relay.answer_submissions_with(answer); + let block = signed_blinded(&world, 1, None); + let (status, json) = post_blinded(publish_app(&world), &block).await; + assert_eq!(status, expected, "{answer}: {json}"); + } + } + + #[tokio::test] + async fn without_a_builder_a_blinded_block_is_a_server_error() { + let world = builder_world(true, SyncStatus::Synced).await; + let network: RpcToP2PRef = Arc::new(RecordingNetwork::default()); + let app = routes() + .with_state(world.store.clone()) + .layer(Extension(network)); + let block = signed_blinded(&world, 1, None); + let (status, _) = post_blinded(app, &block).await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + } } From 706e33d31c9d77e029c2c55569fa186bd3843900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:48:39 -0300 Subject: [PATCH 09/20] docs: describe the fulu builder flow produceBlockV3's builder path, the blinded_blocks endpoint and the slot guard both publish endpoints share, the new metrics, and the one deviation from builder-specs: the bid's fee recipient is not checked, since relays pay the proposer with a transaction instead. --- docs/metrics.md | 3 +++ docs/rpc.md | 47 ++++++++++++++++++++++++++++++++++++++--- docs/spec_deviations.md | 13 +++++++++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 5e3edce48..bd7d30b31 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -469,6 +469,9 @@ ethlambda-specific, not part of the leanMetrics spec. | `lean_beacon_publish_data_columns_seconds` | Histogram | Time to compute and verify a published block's cells and build its data column sidecars | On every published block that carries blobs and whose cells verify | | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5 | | `lean_beacon_builder_request_seconds` | Histogram | Time a builder API call took | On every `getHeader`, `registerValidator`, `submitBlindedBlockV2` and `status` call to the configured builder | call=get_header,register_validators,submit_blinded_block,status
outcome=ok,no_bid,timeout,unreachable,bad_request,unexpected_status,invalid_response,unsupported_fork,encode | 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 2, 5, 10 | | `lean_beacon_builder_registrations_total` | Counter | Validator registrations received, by what became of them | On every `register_validator` request | outcome=forwarded,failed,dropped | | +| `lean_beacon_builder_bid_rejections_total` | Counter | Builder bids refused before selection, by reason | On every bid that fails validation or the state transition | reason=zero_value,parent_hash,prev_randao,timestamp,withdrawals,too_many_blobs,blob_gas,gas_limit,signature,state_transition | | +| `lean_beacon_builder_payload_source_total` | Counter | Blocks produced with a builder configured, by where the payload came from | On every produced block when `--builder` is set | source=local,builder | | +| `lean_beacon_builder_fallback_total` | Counter | Proposals the circuit breaker kept from the builder, by reason | On every proposal by a registered proposer that the breaker trips for | reason=syncing,optimistic_head,no_finality,consecutive_skips,skips_per_epoch | | ### Beacon Committee Cache diff --git a/docs/rpc.md b/docs/rpc.md index e63a27332..b948f8367 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -265,10 +265,11 @@ surface rather than sitting beside it; a `/lean/v0` path on a beacon node is a | `GET` | `/eth/v1/validator/sync_committee_contribution` | JSON | The pooled messages for `(slot, subcommittee_index, beacon_block_root)`, aggregated | | `POST` | `/eth/v1/validator/contribution_and_proofs` | *(status only)* | Validate, pool and gossip `SignedContributionAndProof`s | | `POST` | `/eth/v1/validator/sync_committee_subscriptions` | *(status only)* | Join sync committee subnets until an epoch | -| `GET` | `/eth/v3/validator/blocks/{slot}` | SSZ or JSON | An unsigned fulu block built on the head (`produceBlockV3`) | +| `GET` | `/eth/v3/validator/blocks/{slot}` | SSZ or JSON | An unsigned fulu block built on the head, or, with `--builder`, the blinded block around a winning bid (`produceBlockV3`) | | `POST` | `/eth/v4/validator/blocks/{slot}` | SSZ or JSON | An unsigned gloas block: self-built, with its envelope and blobs when asked, or on a pooled builder bid (`produceBlockV4`) | | `GET` | `/eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}` | SSZ or JSON | The unsigned envelope `produceBlockV4` built (gloas) | | `POST` | `/eth/v2/beacon/blocks` | *(status only)* | Gossip and import a signed fulu or gloas block (`publishBlockV2`; JSON or SSZ body) | +| `POST` | `/eth/v2/beacon/blinded_blocks` | *(status only)* | Hand a signed fulu blinded block to the builder to reveal (`publishBlindedBlockV2`; JSON or SSZ body) | | `POST` | `/eth/v1/beacon/execution_payload_envelopes` | *(status only)* | Gossip a signed envelope and its data columns (gloas; JSON or SSZ body) | | `POST` | `/eth/v1/beacon/execution_payload_bids` | *(status only)* | Validate, pool and gossip a builder's `SignedExecutionPayloadBid` (gloas; JSON or SSZ body) | | `POST` | `/eth/v1/validator/proposer_preferences` | *(status only)* | Validate, cache and gossip `SignedProposerPreferences` (gloas; JSON or SSZ body) | @@ -437,12 +438,35 @@ the chain actor writes, so no request waits on the actor. nothing verifiable is pooled). The payload bundle's KZG commitments go in the body. The state root comes from running the block through `process_block`. The answer is fulu `BlockContents`: the block, the bundle's cell proofs and its blobs, with - `Eth-Execution-Payload-Blinded: false`; there is no builder flow. It is a + `Eth-Execution-Payload-Blinded: false`. It is a **`503`** without a configured execution client, or when the bundle is malformed (its commitments, proofs and blobs disagree in count). A slot the schedule does not place at fulu (gloas included) is a `400`, checked before the execution client; `produceBlockV3` is the fulu endpoint, and a gloas slot goes to `produceBlockV4`. + + **The builder path** is taken only when `--builder` is set, the proposer + registered through this node (`register_validator`, remembered for two + epochs after it was last sent), and the circuit breaker is quiet: it keeps + the builder out while the node is syncing, the head is optimistic, finality + is too far behind, or too many slots were skipped (the `--builder-fallback-*` + flags set the thresholds). The builder's `getHeader` is then asked + concurrently with the local build, bounded by its own timeout, and an error + or a timeout means no bid. A bid is checked before it competes: a non-zero + value, the head's execution block as its parent, the proposal's `prev_randao` + and timestamp, the withdrawals root, the blob commitment and blob gas bounds, + a gas limit within one step of the parent's toward the registered target, + and the builder's signature. Its fee recipient is not checked (see + `spec_deviations.md`). A bid that passes goes through `assemble_blinded_block` + with the packed attestations, operations and sync aggregate, so the blinded + block's state root comes from running the block through `process_block`. + `builder_boost_factor` (percent, default 100) then weighs the two: the local + payload wins iff its value is at least `factor * bid / 100`, so a tie and a + factor of 0 keep it and `u64::MAX` takes the bid; the execution client's + `shouldOverrideBuilder` keeps the local payload whatever the bid pays. If one + side fails the other is served. A blinded answer is the blinded block as SSZ + or JSON, with `Eth-Execution-Payload-Blinded: true` and + `Eth-Execution-Payload-Value` set to the bid's value in wei. - **Graffiti** is the validator client's, with both clients' codes and commits appended as far as they fit: `hello RH1a2bLA3c4d` for reth at `1a2b…` under ethlambda at `3c4d…`. The execution client's half comes from @@ -474,7 +498,24 @@ the chain actor writes, so no request waits on the actor. slot must be scheduled at gloas, the parent must be held, the slot must be after the parent's, and the proposer and signature are checked against the parent's state advanced to the slot (all `400` otherwise), before the block is - gossiped and handed to the chain actor. + gossiped and handed to the chain actor. This node publishes one block per + slot: a second block for a slot whose root differs from the first, through + this endpoint or `blinded_blocks`, is a `400`, since a validator that signed + two would be slashed. The same block again is a retry and passes. +- **`POST beacon/blinded_blocks`** takes a fulu `SignedBlindedBeaconBlock` as + JSON or SSZ, by `Content-Type` as above, with `Eth-Consensus-Version: fulu`. + It is checked as a fulu `beacon/blocks` is, before anything leaves this node: + the slot is scheduled at fulu, the parent is held and before the block, the + proposer is the one the parent's state names for the slot, and the proposer's + signature verifies (a blinded block has its full block's root, so the check + is the same). Then the slot is claimed, shared with `beacon/blocks`, and the + block goes to the builder (`submitBlindedBlockV2`) on its own task, so a + validator client that hangs up cannot cancel the reveal. The builder + publishes the full block and its data columns; it returns over gossip and + imports like any block, and nothing is gossiped here. The status is the + builder's: 202 becomes `200`, a builder `400` stays `400`, and anything else + is a `500`. There is no local fallback, since a second block would be an + equivocation. With no `--builder` the answer is `500`. - **The operation pool** holds, beside the attestation pool and in memory in the same `Store` (`Store::operation_pool`): proposer slashings (one per proposer), attester slashings (one per `hash_tree_root`), voluntary exits and diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 191be46cd..8e04fb262 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -125,7 +125,18 @@ from the Beacon API, and no other way. the bid's. - **Why:** gossip and API bids need only a pool and the gossip rules. A builder API client is a separate crate with its own failure modes and timeouts, and - is left for a later phase. + is left for a later phase for gloas. Fulu's builder API (MEV-Boost) is now + served (`produceBlockV3` and `blinded_blocks`); gloas's is not. + +## Builder bids: the fee recipient is not checked + +- **The specification:** builder-specs' `process_bid` asserts + `header.fee_recipient == fee_recipient`. +- **ethlambda:** a mismatch is logged at `debug` only. +- **Why:** relays pay the proposer with a transaction in the payload and name + the builder's address as the coinbase, so the assertion would refuse nearly + every real bid. Every client reviewed (Lighthouse, Teku, Prysm, Nimbus, + Lodestar, Grandine) skips it. ## Fee recipient and gas target come from proposer preferences, with fallbacks From 2e25eb81e624fc4a8c3d732555364a91ba60f877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:00:38 -0300 Subject: [PATCH 10/20] fix(rpc): close the slot-claim and bid-check gaps in the builder flow A claim below the finalized slot was inserted and then pruned, so the same slot could be claimed again with a different root; it is now refused. The blinded_blocks route gets the publish body limit, and the builder's answer to a signed block is logged from the submission task, which survives a validator client hanging up. A bid's block number is checked against the parent's, which is always known. produceBlockV3 now checks the local payload and the bid without a state transition, picks one, and builds only that block, falling back to the other only if the first fails to build. Tests now assert the blinded header, the value and the number of getHeader calls, and cover the cross-endpoint guard, a failed submit, an unreachable relay and both sides failing, plus the circuit breaker's edges, bid signature variants and the fallback flags. --- bin/ethlambda/src/cli.rs | 28 + crates/net/rpc/src/beacon/blinded_blocks.rs | 104 +++- crates/net/rpc/src/beacon/builder_flow/bid.rs | 47 +- .../rpc/src/beacon/builder_flow/breaker.rs | 40 ++ .../net/rpc/src/beacon/builder_flow/cache.rs | 2 +- .../net/rpc/src/beacon/builder_flow/choice.rs | 6 +- crates/net/rpc/src/beacon/proposal.rs | 544 +++++++++++++----- 7 files changed, 586 insertions(+), 185 deletions(-) diff --git a/bin/ethlambda/src/cli.rs b/bin/ethlambda/src/cli.rs index 72ab02014..5fc1a57f1 100644 --- a/bin/ethlambda/src/cli.rs +++ b/bin/ethlambda/src/cli.rs @@ -1106,6 +1106,34 @@ mod tests { assert_eq!(beacon.mainnet.builder_fallback_epochs_since_finalization, 2); } + #[test] + fn each_fallback_flag_needs_a_builder() { + for flag in [ + "--builder-fallback-skips", + "--builder-fallback-skips-per-epoch", + "--builder-fallback-epochs-since-finalization", + ] { + let mut args = beacon_args(); + args.extend([flag, "5"]); + try_parse_from(args).expect_err(flag); + } + } + + #[test] + fn a_bad_fallback_value_is_refused() { + for flag in [ + "--builder-fallback-skips", + "--builder-fallback-skips-per-epoch", + "--builder-fallback-epochs-since-finalization", + ] { + for value in ["-1", "many"] { + let mut args = beacon_args(); + args.extend(["--builder", "http://127.0.0.1:18550", flag, value]); + try_parse_from(args).expect_err(flag); + } + } + } + #[test] fn a_zero_header_timeout_is_refused() { let mut args = beacon_args(); diff --git a/crates/net/rpc/src/beacon/blinded_blocks.rs b/crates/net/rpc/src/beacon/blinded_blocks.rs index 82f73f733..e5a686831 100644 --- a/crates/net/rpc/src/beacon/blinded_blocks.rs +++ b/crates/net/rpc/src/beacon/blinded_blocks.rs @@ -35,14 +35,45 @@ use super::{ApiError, BodyEncoding}; #[derive(Debug, Clone, Default)] pub(crate) struct PublishedProposals(Arc>>); +/// Why a slot could not be claimed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ClaimError { + /// The slot is below the finalized one, where claims are forgotten, so a + /// second block could not be told from the first. + Finalized, + /// This node already published another block for the slot. + Taken(Root), +} + +impl ClaimError { + /// The 400 a publish endpoint answers with. + pub(crate) fn into_api_error(self) -> ApiError { + match self { + Self::Finalized => ApiError::BadRequest("the slot is already finalized"), + Self::Taken(_) => { + ApiError::BadRequest("this node already published a different block for the slot") + } + } + } +} + impl PublishedProposals { - /// Claims `slot` for `root`, or refuses with the root already claimed. - /// The same root again is a retry, not an equivocation. - pub(crate) fn claim(&self, slot: Slot, root: Root, finalized_slot: Slot) -> Result<(), Root> { + /// Claims `slot` for `root`, or refuses: below `finalized_slot` (a slot + /// equal to it is still held, as `split_off` keeps it), or with the root + /// already claimed. The same root again is a retry, not an equivocation. + pub(crate) fn claim( + &self, + slot: Slot, + root: Root, + finalized_slot: Slot, + ) -> Result<(), ClaimError> { + if slot < finalized_slot { + return Err(ClaimError::Finalized); + } let mut published = self.0.lock().expect("published proposals lock poisoned"); *published = published.split_off(&finalized_slot); match published.get(&slot) { - Some(&claimed) if claimed != root => Err(claimed), + Some(&claimed) if claimed != root => Err(ClaimError::Taken(claimed)), Some(_) => Ok(()), None => { published.insert(slot, root); @@ -130,31 +161,36 @@ pub(crate) async fn post_blinded_block( return ApiError::Internal("checking the blinded block failed").into_response(); } }; - if published.claim(slot, root, finalized_slot(&store)).is_err() { - warn!(slot, "Refusing a second, different block for the slot"); - return ApiError::BadRequest("this node already published a different block for the slot") - .into_response(); + if let Err(err) = published.claim(slot, root, finalized_slot(&store)) { + warn!(slot, ?err, "Refusing a block for the slot"); + return err.into_api_error().into_response(); } + // The outcome is logged inside the task: once the proposer has signed, a + // validator client that hangs up drops this handler, and the reveal's + // result would otherwise go unrecorded. A block the builder does not take + // is a missed slot, so those are errors. let client = builder.client.clone(); - let submitted = - tokio::spawn(async move { client.submit_blinded_block(&signed, ForkName::Fulu).await }) - .await; - match submitted { - Ok(Ok(())) => { - info!(slot, "The builder took the blinded block"); - StatusCode::OK.into_response() + let submitted = tokio::spawn(async move { + let result = client.submit_blinded_block(&signed, ForkName::Fulu).await; + match &result { + Ok(()) => info!(slot, "The builder took the blinded block"), + Err(BuilderError::BadRequest(message)) => { + error!(slot, %message, "The builder refused the signed blinded block") + } + Err(err) => error!(slot, %err, "The builder did not take the signed blinded block"), } + result + }) + .await; + match submitted { + Ok(Ok(())) => StatusCode::OK.into_response(), Ok(Err(BuilderError::BadRequest(message))) => { - warn!(slot, %message, "The builder refused the blinded block"); ApiError::BadRequestDetail(message).into_response() } - Ok(Err(err)) => { - warn!(slot, %err, "The builder did not take the blinded block"); - ApiError::Internal("the builder did not take the block").into_response() - } + Ok(Err(_)) => ApiError::Internal("the builder did not take the block").into_response(), Err(err) => { - error!(%err, "Submitting the blinded block failed"); + error!(slot, %err, "Submitting the blinded block failed"); ApiError::Internal("submitting the block failed").into_response() } } @@ -170,13 +206,33 @@ mod tests { let first = Root::repeat_byte(1); assert_eq!(published.claim(10, first, 0), Ok(())); assert_eq!(published.claim(10, first, 0), Ok(())); - assert_eq!(published.claim(10, Root::repeat_byte(2), 0), Err(first)); + assert_eq!( + published.claim(10, Root::repeat_byte(2), 0), + Err(ClaimError::Taken(first)) + ); } #[test] - fn claims_below_the_finalized_slot_are_forgotten() { + fn a_claim_below_the_finalized_slot_is_refused() { let published = PublishedProposals::default(); published.claim(10, Root::repeat_byte(1), 0).unwrap(); - assert_eq!(published.claim(10, Root::repeat_byte(2), 11), Ok(())); + assert_eq!( + published.claim(10, Root::repeat_byte(2), 11), + Err(ClaimError::Finalized) + ); + assert_eq!( + published.claim(10, Root::repeat_byte(1), 11), + Err(ClaimError::Finalized) + ); + } + + #[test] + fn the_finalized_slot_itself_can_still_be_claimed() { + let published = PublishedProposals::default(); + assert_eq!(published.claim(11, Root::repeat_byte(1), 11), Ok(())); + assert_eq!( + published.claim(11, Root::repeat_byte(2), 11), + Err(ClaimError::Taken(Root::repeat_byte(1))) + ); } } diff --git a/crates/net/rpc/src/beacon/builder_flow/bid.rs b/crates/net/rpc/src/beacon/builder_flow/bid.rs index 7e047815c..88686a98f 100644 --- a/crates/net/rpc/src/beacon/builder_flow/bid.rs +++ b/crates/net/rpc/src/beacon/builder_flow/bid.rs @@ -27,6 +27,8 @@ pub(crate) const GAS_PER_BLOB: u64 = 1 << 17; pub(crate) struct BidContext<'a> { pub(crate) genesis_fork_version: Version, pub(crate) inputs: &'a PayloadInputs, + /// The block number of the payload the block builds on. + pub(crate) parent_block_number: u64, /// The gas limit of the payload the block builds on. pub(crate) parent_gas_limit: u64, /// The gas limit the proposer registered. @@ -41,6 +43,7 @@ pub(crate) enum Rejection { ParentHash, PrevRandao, Timestamp, + BlockNumber, Withdrawals, TooManyBlobs, BlobGas, @@ -56,6 +59,7 @@ impl Rejection { Self::ParentHash => "parent_hash", Self::PrevRandao => "prev_randao", Self::Timestamp => "timestamp", + Self::BlockNumber => "block_number", Self::Withdrawals => "withdrawals", Self::TooManyBlobs => "too_many_blobs", Self::BlobGas => "blob_gas", @@ -67,6 +71,10 @@ impl Rejection { /// Whether `bid` is acceptable for the proposal `context` describes. /// Cheap checks first; the signature, the one that costs a pairing, last. +/// +/// The bid's fork label is not checked here: an electra bid at a fulu slot is +/// accepted on purpose (the containers are identical, so a lagging relay's bid +/// stays usable), and `BuilderClient` already refuses every other fork. pub(crate) fn validate(bid: &SignedBuilderBid, context: &BidContext) -> Result<(), Rejection> { let message = &bid.message; let header = &message.header; @@ -82,6 +90,11 @@ pub(crate) fn validate(bid: &SignedBuilderBid, context: &BidContext) -> Result<( if header.timestamp != context.inputs.timestamp { return Err(Rejection::Timestamp); } + // Against the parent's number rather than the local payload's, unlike + // Lighthouse: the parent is always known, even when the local build failed. + if context.parent_block_number.checked_add(1) != Some(header.block_number) { + return Err(Rejection::BlockNumber); + } let withdrawals: capella::Withdrawals = context .inputs .withdrawals @@ -123,10 +136,13 @@ mod tests { BlsPubkey, Bytes32, ExecutionAddress, ExecutionBlockHash, KzgCommitment, Uint256, }; + use ethlambda_state_transition::beacon::helpers::test_state::secret_key_for; + use super::*; use crate::beacon::builder_client::mock_relay::signed_bid; const GENESIS_FORK_VERSION: Version = [0x10, 0x00, 0x00, 0x38]; + const PARENT_BLOCK_NUMBER: u64 = 99; const PARENT_GAS_LIMIT: u64 = 30_000_000; const TARGET_GAS_LIMIT: u64 = 40_000_000; /// Lighthouse's `expected_gas_limit(30M, 40M)`: one adjustment step up. @@ -150,6 +166,7 @@ mod tests { BidContext { genesis_fork_version: GENESIS_FORK_VERSION, inputs, + parent_block_number: PARENT_BLOCK_NUMBER, parent_gas_limit: PARENT_GAS_LIMIT, target_gas_limit: TARGET_GAS_LIMIT, max_blobs: 9, @@ -166,6 +183,7 @@ mod tests { header.parent_hash = inputs.parent_hash; header.prev_randao = inputs.prev_randao; header.timestamp = inputs.timestamp; + header.block_number = PARENT_BLOCK_NUMBER + 1; header.withdrawals_root = withdrawals.hash_tree_root(); header.gas_limit = NEXT_GAS_LIMIT; header.blob_gas_used = GAS_PER_BLOB; @@ -192,7 +210,7 @@ mod tests { #[test] fn each_check_refuses_its_own_field() { type Edit = fn(&mut BuilderBid); - let cases: [(Rejection, Edit); 8] = [ + let cases: [(Rejection, Edit); 9] = [ (Rejection::ZeroValue, |bid| bid.value = Uint256::from(0u64)), (Rejection::ParentHash, |bid| { bid.header.parent_hash = ExecutionBlockHash::repeat_byte(9) @@ -201,6 +219,7 @@ mod tests { bid.header.prev_randao = Bytes32::repeat_byte(9) }), (Rejection::Timestamp, |bid| bid.header.timestamp += 1), + (Rejection::BlockNumber, |bid| bid.header.block_number += 1), (Rejection::Withdrawals, |bid| { bid.header.withdrawals_root = Bytes32::repeat_byte(9) }), @@ -219,12 +238,36 @@ mod tests { } #[test] - fn a_bid_signed_by_another_key_is_refused() { + fn a_block_number_that_overflows_is_refused() { + let inputs = inputs(); + let mut context = context(&inputs); + context.parent_block_number = u64::MAX; + let bid = bid(|bid| bid.header.block_number = 0); + assert_eq!(validate(&bid, &context), Err(Rejection::BlockNumber)); + } + + #[test] + fn a_bid_changed_after_signing_is_refused() { let mut forged = bid(|_| {}); forged.message.value = Uint256::from(2_000_000_000u64); assert_eq!(rejection(&forged), Some(Rejection::Signature)); } + #[test] + fn a_bid_signed_by_a_key_other_than_the_one_it_names_is_refused() { + let mut forged = bid(|_| {}); + // The signature is the relay's, the message names another key. + forged.message.pubkey = BlsPubkey(secret_key_for(7).sk_to_pk().to_bytes()); + assert_eq!(rejection(&forged), Some(Rejection::Signature)); + } + + #[test] + fn a_bid_signed_under_another_genesis_fork_version_is_refused() { + let message = bid(|_| {}).message; + let wrong = signed_bid(message, [0x99, 0x00, 0x00, 0x99]); + assert_eq!(rejection(&wrong), Some(Rejection::Signature)); + } + /// The gloas gossip predicate this reuses agrees with Lighthouse's /// `expected_gas_limit` on its own test vectors. #[test] diff --git a/crates/net/rpc/src/beacon/builder_flow/breaker.rs b/crates/net/rpc/src/beacon/builder_flow/breaker.rs index 898e4385c..07f5129b9 100644 --- a/crates/net/rpc/src/beacon/builder_flow/breaker.rs +++ b/crates/net/rpc/src/beacon/builder_flow/breaker.rs @@ -46,6 +46,7 @@ impl Default for FallbackThresholds { pub(crate) enum Trip { Syncing, OptimisticHead, + /// Reads the state's realized `finalized_checkpoint`, not fork choice's. NoFinality, ConsecutiveSkips, SkipsPerEpoch, @@ -183,6 +184,45 @@ mod tests { assert_eq!(check_at(&state_at(200, &[], 2)), Err(Trip::NoFinality)); } + #[test] + fn a_proposal_near_genesis_counts_no_slot_before_slot_one() { + assert_eq!(check_at(&state_at(1, &[], 0)), Ok(())); + assert_eq!(check_at(&state_at(5, &[], 0)), Ok(())); + // Slots 1 to 4 are all empty: four in a row, and slot 0 is never judged. + assert_eq!( + check_at(&state_at(5, &[1, 2, 3, 4], 0)), + Err(Trip::ConsecutiveSkips) + ); + assert_eq!(check_at(&state_at(4, &[1, 2, 3], 0)), Ok(())); + } + + #[test] + fn a_history_that_wraps_is_read_at_its_wrapped_index() { + let wrapped = 8192 + 100; + assert_eq!(check_at(&state_at(wrapped, &[], 300)), Ok(())); + let run = [wrapped - 4, wrapped - 3, wrapped - 2, wrapped - 1]; + assert_eq!( + check_at(&state_at(wrapped, &run, 300)), + Err(Trip::ConsecutiveSkips) + ); + } + + #[test] + fn a_slot_older_than_the_history_counts_as_filled() { + let slot = 8192 + 100; + let all_empty: Vec = (1..slot).collect(); + let state = state_at(slot, &all_empty, 300); + // Slot 1 is out of the state's reach, so its roots are not compared. + assert!(!is_empty(&state, 1)); + assert!(is_empty(&state, slot - 1)); + } + + #[test] + fn an_empty_run_that_does_not_end_right_before_the_proposal_is_not_consecutive() { + // Five empty slots, but slot 99 had a block. + assert_eq!(check_at(&state_at(100, &[90, 91, 92, 93, 94], 2)), Ok(())); + } + #[test] fn an_optimistic_head_or_a_syncing_node_keeps_the_builder_out() { let state = state_at(100, &[], 2); diff --git a/crates/net/rpc/src/beacon/builder_flow/cache.rs b/crates/net/rpc/src/beacon/builder_flow/cache.rs index f0d0903a2..8ccbb7ecb 100644 --- a/crates/net/rpc/src/beacon/builder_flow/cache.rs +++ b/crates/net/rpc/src/beacon/builder_flow/cache.rs @@ -72,7 +72,7 @@ impl BuilderRegistrations { } fn expired(entry: &Entry, epoch: Epoch) -> bool { - epoch > entry.seen + REGISTRATION_LIFETIME_EPOCHS + epoch > entry.seen.saturating_add(REGISTRATION_LIFETIME_EPOCHS) } #[cfg(test)] diff --git a/crates/net/rpc/src/beacon/builder_flow/choice.rs b/crates/net/rpc/src/beacon/builder_flow/choice.rs index 7d76ca27c..fd5712b10 100644 --- a/crates/net/rpc/src/beacon/builder_flow/choice.rs +++ b/crates/net/rpc/src/beacon/builder_flow/choice.rs @@ -34,9 +34,9 @@ pub(crate) struct LocalOffer { /// beacon-APIs' `builder_boost_factor`, taken literally: the local payload /// wins iff `local >= factor * (bid / 100)`. So 100 compares the two as they /// are, 0 always keeps the local payload, and a tie goes local. -/// `u64::MAX` always takes the bid. The execution client's -/// `shouldOverrideBuilder` beats any bid. When one side failed, the other -/// stands. +/// A factor of `u64::MAX` always takes the builder, except that the execution +/// client's `shouldOverrideBuilder` still wins. When one side failed, the +/// other stands. pub(crate) fn choose( local: Option, bid_value: Option, diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index 315c7dc57..a14f29e79 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -51,13 +51,13 @@ use ethlambda_state_transition::beacon::{ use ethlambda_storage::Store; use ethlambda_types::{ beacon::{ - builder::{BlindedBeaconBlock, ValidatorRegistrationV1}, + builder::{BlindedBeaconBlock, BuilderBid, ValidatorRegistrationV1}, config::Config, containers::{ self, BeaconState, altair::SyncAggregate, deneb::Blob, - electra::{Attestation, BeaconBlock, SignedBeaconBlock}, + electra::{Attestation, BeaconBlock, ExecutionRequests, SignedBeaconBlock}, }, fork::ForkName, preset, @@ -129,7 +129,8 @@ pub(crate) fn routes() -> Router { ) .route( "/eth/v2/beacon/blinded_blocks", - post(blinded_blocks::post_blinded_block), + post(blinded_blocks::post_blinded_block) + .layer(DefaultBodyLimit::max(super::MAX_PUBLISH_BODY_BYTES)), ) .layer(Extension(PublishedProposals::default())) } @@ -253,16 +254,10 @@ async fn post_block( } }; let root = signed_block.message.hash_tree_root(); - if published - .claim(signed_block.message.slot, root, finalized_slot(&store)) - .is_err() - { - warn!( - slot = signed_block.message.slot, - "Refusing a second, different block for the slot" - ); - return ApiError::BadRequest("this node already published a different block for the slot") - .into_response(); + let slot = signed_block.message.slot; + if let Err(err) = published.claim(slot, root, finalized_slot(&store)) { + warn!(slot, ?err, "Refusing a block for the slot"); + return err.into_api_error().into_response(); } let block = containers::SignedBeaconBlock::Fulu(signed_block); let sidecars = sidecars @@ -338,20 +333,11 @@ async fn post_gloas_block( Ok(Err(err)) => return err.into_response(), Err(_) => return ApiError::Internal("verifying the block failed").into_response(), } - if published - .claim( - block.slot(), - block.message_hash_tree_root(), - finalized_slot(store), - ) - .is_err() - { - warn!( - slot = block.slot(), - "Refusing a second, different block for the slot" - ); - return ApiError::BadRequest("this node already published a different block for the slot") - .into_response(); + let slot = block.slot(); + let root = block.message_hash_tree_root(); + if let Err(err) = published.claim(slot, root, finalized_slot(store)) { + warn!(slot, ?err, "Refusing a block for the slot"); + return err.into_api_error().into_response(); } // A gloas block carries no blobs of its own: its data columns travel with // the execution payload envelope, published separately, so there are no @@ -565,11 +551,13 @@ impl Packed { } } -/// The local block and what its payload offers. -struct LocalBlock { - contents: FuluBlockContents, - value: Uint256, - should_override_builder: bool, +/// The local payload, checked and converted for a block but not yet built +/// into one, so choosing between it and a bid costs no state transition. +struct LocalParts { + built: BuiltPayload, + blobs: Blobs, + kzg_proofs: CellKzgProofs, + execution_requests: ExecutionRequests, } /// The blinded block around a validated bid, and what the bid pays. @@ -591,8 +579,10 @@ struct BuilderRequest<'a> { /// /// With a builder layered, the proposer registered through this node and the /// circuit breaker quiet, the builder's bid is fetched alongside the local -/// payload. Whichever [`choice::choose`] picks goes into the block. Otherwise -/// the block is built exactly as it is without a builder. +/// payload. Both are checked without a state transition, [`choice::choose`] +/// picks one, and only that one is built into a block; the other is built only +/// if the first fails. Without a builder the block is built exactly as it is +/// without one. #[allow(clippy::too_many_arguments)] async fn produce( store: &Store, @@ -632,63 +622,91 @@ async fn produce( ); let graffiti = graffiti::with_client_versions(graffiti, el_version.as_ref(), own_version); let packed = pack(store, sync_pool, &state, head_root, slot, &config); - let local = built.and_then(|built| { - local_block( - &state, - built, - &packed, - randao_reveal, - graffiti, - &config, - options, - slot, - ) - }); - let blinded = request.as_ref().zip(bid).and_then(|(request, bid)| { - blinded_block( - &state, - &config, - request, - bid, - &packed, - randao_reveal, - graffiti, - options, - ) + let local = built.and_then(|built| local_parts(built, slot)); + let mut bid = request + .as_ref() + .zip(bid) + .and_then(|(request, bid)| accepted_bid(&state, &config, request, bid)); + + let local_offer = local.as_ref().ok().map(|parts| LocalOffer { + value: wei_u128(&parts.built.block_value), + should_override_builder: parts.built.should_override_builder, }); + let bid_value = bid.as_ref().map(|bid| wei_u128(&bid.value)); + let order = if choice::choose(local_offer, bid_value, boost_factor) == Some(Source::Builder) { + [Source::Builder, Source::Local] + } else { + [Source::Local, Source::Builder] + }; - let local_offer = local.as_ref().ok().map(|local| LocalOffer { - value: wei_u128(&local.value), - should_override_builder: local.should_override_builder, - }); - let bid_value = blinded.as_ref().map(|blinded| wei_u128(&blinded.value)); - if choice::choose(local_offer, bid_value, boost_factor) == Some(Source::Builder) - && let Some(blinded) = blinded - { - inc_payload_source(Source::Builder.label()); - let value = decimal(&blinded.value); - info!(%slot, proposer, %value, "Produced a blinded block around the builder's bid"); - return Ok((Produced::Blinded(blinded.block), value, fork)); + let (mut local, mut local_error) = match local { + Ok(parts) => (Some(parts), None), + Err(err) => (None, Some(err)), + }; + for source in order { + match source { + Source::Builder => { + let Some(message) = bid.take() else { + continue; + }; + let blinded = blinded_block( + &state, + &config, + message, + &packed, + randao_reveal, + graffiti, + options, + ); + if let Some(blinded) = blinded { + inc_payload_source(Source::Builder.label()); + let value = decimal(&blinded.value); + info!(%slot, proposer, %value, "Produced a blinded block around the builder's bid"); + return Ok((Produced::Blinded(blinded.block), value, fork)); + } + } + Source::Local => { + let Some(parts) = local.take() else { + continue; + }; + let assembled = local_block( + &state, + parts, + &packed, + randao_reveal, + graffiti, + &config, + options, + slot, + ); + match assembled { + Ok((contents, value)) => { + if builder.is_some() { + inc_payload_source(Source::Local.label()); + } + let block = &contents.block; + info!( + %slot, + proposer, + attestations = block.body.attestations.len(), + slashings = block.body.proposer_slashings.len() + + block.body.attester_slashings.len(), + exits = block.body.voluntary_exits.len(), + bls_changes = block.body.bls_to_execution_changes.len(), + transactions = block.body.execution_payload.transactions.len(), + blobs = block.body.blob_kzg_commitments.len(), + graffiti = %graffiti::display(&block.body.graffiti), + "Produced block" + ); + return Ok((Produced::Full(contents), decimal(&value), fork)); + } + Err(err) => local_error = Some(err), + } + } + } } // The local error surfaces only when the builder did not step in. - let local = local?; - if builder.is_some() { - inc_payload_source(Source::Local.label()); - } - let block = &local.contents.block; - info!( - %slot, - proposer, - attestations = block.body.attestations.len(), - slashings = block.body.proposer_slashings.len() + block.body.attester_slashings.len(), - exits = block.body.voluntary_exits.len(), - bls_changes = block.body.bls_to_execution_changes.len(), - transactions = block.body.execution_payload.transactions.len(), - blobs = block.body.blob_kzg_commitments.len(), - graffiti = %graffiti::display(&block.body.graffiti), - "Produced block" - ); - Ok((Produced::Full(local.contents), decimal(&local.value), fork)) + Err(local_error.unwrap_or(ApiError::Internal("the block failed to build"))) } /// The block's consensus contents for `state`: attestations, operations and @@ -731,19 +749,10 @@ fn pack( } } -/// The local block: the execution client's payload around `packed`, with -/// its blobs, cell proofs and value. -#[allow(clippy::too_many_arguments)] -fn local_block( - state: &BeaconState, - mut built: BuiltPayload, - packed: &Packed, - randao_reveal: BlsSignature, - graffiti: Bytes32, - config: &Config, - options: BlockProcessingOptions, - slot: Slot, -) -> Result { +/// The execution client's payload checked and converted for a block: its +/// blobs bundle against the commitments, the blobs and cell proofs into their +/// containers, and its execution requests parsed. No state transition runs. +fn local_parts(mut built: BuiltPayload, slot: Slot) -> Result { let bundle = &built.blobs_bundle; let malformed = bundle.commitments.len() != bundle.blobs.len() || bundle.proofs.len() != bundle.blobs.len() * preset::CELLS_PER_EXT_BLOB; @@ -776,7 +785,33 @@ fn local_block( }; let execution_requests = parse_execution_requests(&built.execution_requests) .map_err(|_| ApiError::Internal("the execution client's request list is malformed"))?; + Ok(LocalParts { + built, + blobs, + kzg_proofs, + execution_requests, + }) +} +/// The local block: the execution client's payload around `packed`, with +/// its blobs and cell proofs, and the payload's value. +#[allow(clippy::too_many_arguments)] +fn local_block( + state: &BeaconState, + parts: LocalParts, + packed: &Packed, + randao_reveal: BlsSignature, + graffiti: Bytes32, + config: &Config, + options: BlockProcessingOptions, + slot: Slot, +) -> Result<(FuluBlockContents, Uint256), ApiError> { + let LocalParts { + built, + blobs, + kzg_proofs, + execution_requests, + } = parts; let inputs = |attestations, operations, sync_aggregate| BlockInputs { sync_aggregate, randao_reveal, @@ -814,15 +849,12 @@ fn local_block( } Err(_) => return Err(ApiError::Internal("the block failed to build")), }; - Ok(LocalBlock { - contents: FuluBlockContents { - block, - kzg_proofs, - blobs, - }, - value: built.block_value, - should_override_builder: built.should_override_builder, - }) + let contents = FuluBlockContents { + block, + kzg_proofs, + blobs, + }; + Ok((contents, built.block_value)) } /// What to ask the builder for, if anything. @@ -877,19 +909,13 @@ async fn fetch_bid(request: Option<&BuilderRequest<'_>>, slot: Slot) -> Option Option { +) -> Option { let slot = state.slot(); let BeaconState::Fulu(fulu) = state else { return None; @@ -898,6 +924,7 @@ fn blinded_block( let context = BidContext { genesis_fork_version: config.genesis_fork_version, inputs: &inputs, + parent_block_number: fulu.latest_execution_payload_header.block_number, parent_gas_limit: fulu.latest_execution_payload_header.gas_limit, target_gas_limit: request.registration.gas_limit, max_blobs: config.max_blobs_per_block(compute_epoch_at_slot(slot)), @@ -913,6 +940,22 @@ fn blinded_block( // and name the builder as its fee recipient (docs/spec_deviations.md). debug!(%slot, "The bid's fee recipient is not the registered one"); } + Some(message) +} + +/// The blinded block around an accepted bid, if it passes the state +/// transition. +#[allow(clippy::too_many_arguments)] +fn blinded_block( + state: &BeaconState, + config: &Config, + message: BuilderBid, + packed: &Packed, + randao_reveal: BlsSignature, + graffiti: Bytes32, + options: BlockProcessingOptions, +) -> Option { + let slot = state.slot(); let assemble = |attestations, operations, sync_aggregate| { assemble_blinded_block( state, @@ -1453,6 +1496,7 @@ mod tests { header.parent_hash = inputs.parent_hash; header.prev_randao = inputs.prev_randao; header.timestamp = inputs.timestamp; + header.block_number = fulu.latest_execution_payload_header.block_number + 1; header.withdrawals_root = withdrawals.hash_tree_root(); header.gas_limit = TARGET_GAS_LIMIT.clamp( parent_gas_limit - step, @@ -1486,42 +1530,111 @@ mod tests { } } - async fn produce_json( + /// What a `produceBlockV3` answer says about the payload it carries. + struct Answer { + status: StatusCode, + blinded: Option, + value: Option, + json: serde_json::Value, + } + + impl Answer { + /// The answer was a 200 whose headers and body say `blinded` and + /// `value` (wei, decimal), and the relay was asked `header_requests` + /// times. + fn assert(&self, world: &BuilderWorld, blinded: bool, value: &str, header_requests: usize) { + assert_eq!(self.status, StatusCode::OK, "{}", self.json); + assert_eq!( + self.blinded.as_deref(), + Some(if blinded { "true" } else { "false" }), + "{}", + self.json + ); + assert_eq!(self.value.as_deref(), Some(value), "{}", self.json); + assert_eq!(served_blinded(&self.json), blinded, "{}", self.json); + assert_eq!( + world.relay.header_requests().len(), + header_requests, + "{}", + self.json + ); + } + } + + async fn produce_answer( + world: &BuilderWorld, + engine: Option, + accept: Option<&str>, + query: &str, + ) -> Answer { + let mut request = Request::get(world.uri(query)); + if let Some(accept) = accept { + request = request.header("accept", accept); + } + let response = world + .app(engine) + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + let header = |name: &str| { + response + .headers() + .get(name) + .map(|value| value.to_str().unwrap().to_owned()) + }; + let (status, blinded, value) = ( + response.status(), + header("eth-execution-payload-blinded"), + header("eth-execution-payload-value"), + ); + let body = response.into_body().collect().await.unwrap().to_bytes(); + Answer { + status, + blinded, + value, + json: serde_json::from_slice(&body).unwrap_or_default(), + } + } + + async fn produce( world: &BuilderWorld, engine_value: &'static str, override_builder: bool, query: &str, - ) -> (StatusCode, serde_json::Value) { + ) -> Answer { let engine = super::super::validator_client_tests::fake_execution_client_paying( engine_value, override_builder, ) .await; - let request = Request::get(world.uri(query)).body(Body::empty()).unwrap(); - respond(world.app(Some(engine)), request).await + produce_answer(world, Some(engine), None, query).await } fn served_blinded(json: &serde_json::Value) -> bool { json["execution_payload_blinded"] == serde_json::json!(true) } + /// An execution client that is not there. + fn dead_engine() -> EngineClient { + EngineClient::new( + "http://127.0.0.1:1".to_owned(), + ethlambda_engine::JwtSecret::new([0x0f; 32]), + ) + .unwrap() + } + #[tokio::test] async fn a_bid_worth_more_than_the_local_payload_is_served_blinded() { let world = builder_world(true, SyncStatus::Synced).await; world.offer(ETH, |_| {}); - let (status, json) = produce_json(&world, "0x2a", false, "").await; - assert_eq!(status, StatusCode::OK, "{json}"); - assert!(served_blinded(&json), "{json}"); + let answer = produce(&world, "0x2a", false, "").await; + answer.assert(&world, true, Ð.to_string(), 1); assert_eq!( - json["execution_payload_value"], - serde_json::json!(ETH.to_string()) - ); - assert_eq!( - json["data"]["slot"], + answer.json["data"]["slot"], serde_json::json!(world.slot.to_string()) ); assert_eq!( - json["data"]["body"]["execution_payload_header"]["block_hash"], + answer.json["data"]["body"]["execution_payload_header"]["block_hash"], serde_json::json!(format!("0x{}", "bb".repeat(32))) ); } @@ -1531,17 +1644,16 @@ mod tests { let world = builder_world(true, SyncStatus::Synced).await; world.offer(ETH, |_| {}); // 1 ETH in hex wei. - let (status, json) = produce_json(&world, "0xde0b6b3a7640000", false, "").await; - assert_eq!(status, StatusCode::OK, "{json}"); - assert!(!served_blinded(&json), "{json}"); + let answer = produce(&world, "0xde0b6b3a7640000", false, "").await; + answer.assert(&world, false, Ð.to_string(), 1); } #[tokio::test] async fn a_boost_factor_of_zero_keeps_the_local_payload() { let world = builder_world(true, SyncStatus::Synced).await; world.offer(ETH, |_| {}); - let (_, json) = produce_json(&world, "0x2a", false, "&builder_boost_factor=0").await; - assert!(!served_blinded(&json), "{json}"); + let answer = produce(&world, "0x2a", false, "&builder_boost_factor=0").await; + answer.assert(&world, false, "42", 1); } #[tokio::test] @@ -1549,8 +1661,8 @@ mod tests { let world = builder_world(true, SyncStatus::Synced).await; world.offer(1_000_000_000, |_| {}); let query = format!("&builder_boost_factor={}", u64::MAX); - let (_, json) = produce_json(&world, "0xde0b6b3a7640000", false, &query).await; - assert!(served_blinded(&json), "{json}"); + let answer = produce(&world, "0xde0b6b3a7640000", false, &query).await; + answer.assert(&world, true, "1000000000", 1); } #[tokio::test] @@ -1558,26 +1670,24 @@ mod tests { let world = builder_world(true, SyncStatus::Synced).await; // 10 ETH: the largest round value a u64 of wei holds. world.offer(10 * ETH, |_| {}); - let (_, json) = produce_json(&world, "0x2a", true, "").await; - assert!(!served_blinded(&json), "{json}"); + let answer = produce(&world, "0x2a", true, "").await; + answer.assert(&world, false, "42", 1); } #[tokio::test] async fn an_unregistered_proposer_never_reaches_the_builder() { let world = builder_world(false, SyncStatus::Synced).await; world.offer(ETH, |_| {}); - let (_, json) = produce_json(&world, "0x2a", false, "").await; - assert!(!served_blinded(&json), "{json}"); - assert!(world.relay.header_requests().is_empty()); + let answer = produce(&world, "0x2a", false, "").await; + answer.assert(&world, false, "42", 0); } #[tokio::test] async fn a_syncing_node_keeps_the_builder_out() { let world = builder_world(true, SyncStatus::Syncing).await; world.offer(ETH, |_| {}); - let (_, json) = produce_json(&world, "0x2a", false, "").await; - assert!(!served_blinded(&json), "{json}"); - assert!(world.relay.header_requests().is_empty()); + let answer = produce(&world, "0x2a", false, "").await; + answer.assert(&world, false, "42", 0); } #[tokio::test] @@ -1586,24 +1696,61 @@ mod tests { world.offer(ETH, |bid| { bid.header.parent_hash = ExecutionBlockHash::repeat_byte(9) }); - let (status, json) = produce_json(&world, "0x2a", false, "").await; - assert_eq!(status, StatusCode::OK, "{json}"); - assert!(!served_blinded(&json), "{json}"); + let answer = produce(&world, "0x2a", false, "").await; + answer.assert(&world, false, "42", 1); + } + + #[tokio::test] + async fn a_bid_on_the_wrong_block_number_is_dropped() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |bid| bid.header.block_number += 5); + let answer = produce(&world, "0x2a", false, "").await; + answer.assert(&world, false, "42", 1); } #[tokio::test] async fn without_a_local_payload_the_bid_is_served() { let world = builder_world(true, SyncStatus::Synced).await; world.offer(ETH, |_| {}); - let dead_engine = EngineClient::new( + let answer = produce_answer(&world, Some(dead_engine()), None, "").await; + answer.assert(&world, true, Ð.to_string(), 1); + } + + #[tokio::test] + async fn without_a_bid_or_a_local_payload_there_is_no_block() { + let world = builder_world(true, SyncStatus::Synced).await; + let answer = produce_answer(&world, Some(dead_engine()), None, "").await; + assert_ne!(answer.status, StatusCode::OK, "{}", answer.json); + assert_eq!(world.relay.header_requests().len(), 1); + } + + #[tokio::test] + async fn an_unreachable_builder_leaves_the_local_payload() { + let mut world = builder_world(true, SyncStatus::Synced).await; + // The same registered proposer, with a builder nothing listens at. + let client = BuilderClient::new( "http://127.0.0.1:1".to_owned(), - ethlambda_engine::JwtSecret::new([0x0f; 32]), + Duration::from_secs(1), + "ethlambda/test", ) .unwrap(); - let request = Request::get(world.uri("")).body(Body::empty()).unwrap(); - let (status, json) = respond(world.app(Some(dead_engine)), request).await; - assert_eq!(status, StatusCode::OK, "{json}"); - assert!(served_blinded(&json), "{json}"); + let dead = Arc::new(Builder::new( + client, + NEVER_TRIP, + SyncStatusController::new(SyncStatus::Synced), + )); + let proposer = get_beacon_proposer_index(&world.state).unwrap(); + let registration = ValidatorRegistrationV1 { + fee_recipient: ExecutionAddress::repeat_byte(0x11), + gas_limit: TARGET_GAS_LIMIT, + timestamp: 1, + pubkey: world.state.validator(proposer).unwrap().pubkey, + }; + dead.registrations + .record([®istration], compute_epoch_at_slot(world.slot)); + world.builder = dead; + let answer = produce(&world, "0x2a", false, "").await; + answer.assert(&world, false, "42", 0); } #[tokio::test] @@ -1623,6 +1770,7 @@ mod tests { response.headers()["eth-execution-payload-value"], ETH.to_string().as_str() ); + assert_eq!(world.relay.header_requests().len(), 1); let body = response.into_body().collect().await.unwrap().to_bytes(); let block = BlindedBeaconBlock::from_ssz_bytes(&body).unwrap(); assert_eq!(block.slot, world.slot); @@ -1741,4 +1889,90 @@ mod tests { let (status, _) = post_blinded(app, &block).await; assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); } + + #[tokio::test] + async fn a_claim_survives_a_failed_submission() { + let world = builder_world(true, SyncStatus::Synced).await; + world + .relay + .answer_submissions_with(StatusCode::INTERNAL_SERVER_ERROR); + let app = publish_app(&world); + let first = signed_blinded(&world, 1, None); + let second = signed_blinded(&world, 2, None); + assert_eq!( + post_blinded(app.clone(), &first).await.0, + StatusCode::INTERNAL_SERVER_ERROR + ); + // The slot stays claimed, so a different block is refused outright. + assert_eq!( + post_blinded(app.clone(), &second).await.0, + StatusCode::BAD_REQUEST + ); + // The same block is a retry, and goes to the builder again. + assert_eq!( + post_blinded(app, &first).await.0, + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!(world.relay.blinded_blocks(), [first.clone(), first]); + } + + #[tokio::test] + async fn a_block_published_in_full_blocks_a_different_blinded_one() { + let world = builder_world(true, SyncStatus::Synced).await; + // A full block for the slot from the local path, signed by its proposer. + let answer = produce(&world, "0x2a", false, "").await; + answer.assert(&world, false, "42", 1); + let mut contents: FuluBlockContents = + serde_json::from_value(answer.json["data"].clone()).unwrap(); + // The test store keys its head by a root that is not its header's, so + // the block names the stored key as its parent, as `signed_blinded` does. + let (head_root, _) = super::super::validator::head(&world.store).unwrap(); + contents.block.parent_root = head_root; + let proposer = get_beacon_proposer_index(&world.state).unwrap(); + let domain = get_domain(&world.state, DOMAIN_BEACON_PROPOSER, None); + let signing_root = compute_signing_root(contents.block.hash_tree_root(), domain); + let signed = FuluSignedBlockContents { + signed_block: electra::SignedBeaconBlock { + message: contents.block, + signature: sign_for(proposer as usize, signing_root), + }, + kzg_proofs: contents.kzg_proofs, + blobs: contents.blobs, + }; + + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let app = routes() + .with_state(world.store.clone()) + .layer(Extension(p2p)) + .layer(Extension(world.builder.clone())); + let request = Request::post("/eth/v2/beacon/blocks") + .header("content-type", "application/octet-stream") + .header("eth-consensus-version", "fulu") + .body(Body::from(signed.to_ssz())) + .unwrap(); + let (status, json) = respond(app.clone(), request).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(network.blocks.lock().unwrap().len(), 1); + + let blinded = signed_blinded(&world, 1, None); + let (status, json) = post_blinded(app, &blinded).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert!(world.relay.blinded_blocks().is_empty()); + } + + #[tokio::test] + async fn an_oversized_blinded_block_is_a_413() { + let world = builder_world(true, SyncStatus::Synced).await; + let request = Request::post("/eth/v2/beacon/blinded_blocks") + .header("content-type", "application/octet-stream") + .header("eth-consensus-version", "fulu") + .body(Body::from(vec![ + 0u8; + super::super::MAX_PUBLISH_BODY_BYTES + 1 + ])) + .unwrap(); + let (status, _) = respond(publish_app(&world), request).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + } } From 2aae2a7c2fc0da55f6c328f736d547fcb43fb7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:00:41 -0300 Subject: [PATCH 11/20] docs: note the bid block number check and the publish guard's limits --- docs/metrics.md | 2 +- docs/rpc.md | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index bd7d30b31..087d990f0 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -469,7 +469,7 @@ ethlambda-specific, not part of the leanMetrics spec. | `lean_beacon_publish_data_columns_seconds` | Histogram | Time to compute and verify a published block's cells and build its data column sidecars | On every published block that carries blobs and whose cells verify | | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5 | | `lean_beacon_builder_request_seconds` | Histogram | Time a builder API call took | On every `getHeader`, `registerValidator`, `submitBlindedBlockV2` and `status` call to the configured builder | call=get_header,register_validators,submit_blinded_block,status
outcome=ok,no_bid,timeout,unreachable,bad_request,unexpected_status,invalid_response,unsupported_fork,encode | 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 2, 5, 10 | | `lean_beacon_builder_registrations_total` | Counter | Validator registrations received, by what became of them | On every `register_validator` request | outcome=forwarded,failed,dropped | | -| `lean_beacon_builder_bid_rejections_total` | Counter | Builder bids refused before selection, by reason | On every bid that fails validation or the state transition | reason=zero_value,parent_hash,prev_randao,timestamp,withdrawals,too_many_blobs,blob_gas,gas_limit,signature,state_transition | | +| `lean_beacon_builder_bid_rejections_total` | Counter | Builder bids refused before selection, by reason | On every bid that fails validation or the state transition | reason=zero_value,parent_hash,prev_randao,timestamp,block_number,withdrawals,too_many_blobs,blob_gas,gas_limit,signature,state_transition | | | `lean_beacon_builder_payload_source_total` | Counter | Blocks produced with a builder configured, by where the payload came from | On every produced block when `--builder` is set | source=local,builder | | | `lean_beacon_builder_fallback_total` | Counter | Proposals the circuit breaker kept from the builder, by reason | On every proposal by a registered proposer that the breaker trips for | reason=syncing,optimistic_head,no_finality,consecutive_skips,skips_per_epoch | | diff --git a/docs/rpc.md b/docs/rpc.md index b948f8367..d7e4c0bb9 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -454,7 +454,7 @@ the chain actor writes, so no request waits on the actor. concurrently with the local build, bounded by its own timeout, and an error or a timeout means no bid. A bid is checked before it competes: a non-zero value, the head's execution block as its parent, the proposal's `prev_randao` - and timestamp, the withdrawals root, the blob commitment and blob gas bounds, + and timestamp, the parent's block number plus one, the withdrawals root, the blob commitment and blob gas bounds, a gas limit within one step of the parent's toward the registered target, and the builder's signature. Its fee recipient is not checked (see `spec_deviations.md`). A bid that passes goes through `assemble_blinded_block` @@ -501,7 +501,8 @@ the chain actor writes, so no request waits on the actor. gossiped and handed to the chain actor. This node publishes one block per slot: a second block for a slot whose root differs from the first, through this endpoint or `blinded_blocks`, is a `400`, since a validator that signed - two would be slashed. The same block again is a retry and passes. + two would be slashed. The same block again is a retry and passes. A slot + below the finalized one is refused too, since claims there are forgotten. - **`POST beacon/blinded_blocks`** takes a fulu `SignedBlindedBeaconBlock` as JSON or SSZ, by `Content-Type` as above, with `Eth-Consensus-Version: fulu`. It is checked as a fulu `beacon/blocks` is, before anything leaves this node: @@ -512,7 +513,9 @@ the chain actor writes, so no request waits on the actor. block goes to the builder (`submitBlindedBlockV2`) on its own task, so a validator client that hangs up cannot cancel the reveal. The builder publishes the full block and its data columns; it returns over gossip and - imports like any block, and nothing is gossiped here. The status is the + imports like any block, and nothing is gossiped here. The body limit is + the one `beacon/blocks` has (`413` over it). The builder's answer is logged + from the submission task, an error when it did not take a signed block. The status is the builder's: 202 becomes `200`, a builder `400` stays `400`, and anything else is a `500`. There is no local fallback, since a second block would be an equivocation. With no `--builder` the answer is `500`. From f4c907814ce95217b2d36c9da69898bc1eded6a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:07:36 -0300 Subject: [PATCH 12/20] test(rpc): pin the blinded_blocks body limit and the produce fallback The 413 test passed with the body limit removed, since its body was over axum's default too; a 3 MiB sibling now fails without the layer. Also covers a winning bid whose block and the local block both fail to build, tightens the status assertions of the no-block and unreachable-builder cases, and skips the packing, whose sync aggregate costs a BLS verification, when there is nothing to build a block from. A refused second block now logs the root that holds the slot. --- crates/net/rpc/src/beacon/blinded_blocks.rs | 15 +++++- crates/net/rpc/src/beacon/proposal.rs | 59 ++++++++++++++++++--- docs/rpc.md | 8 +-- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/crates/net/rpc/src/beacon/blinded_blocks.rs b/crates/net/rpc/src/beacon/blinded_blocks.rs index e5a686831..97262ebd2 100644 --- a/crates/net/rpc/src/beacon/blinded_blocks.rs +++ b/crates/net/rpc/src/beacon/blinded_blocks.rs @@ -18,6 +18,7 @@ use ethlambda_state_transition::beacon::block_production::advance_to_slot; use ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index; use ethlambda_state_transition::beacon::stf::verify_proposer_signature; use ethlambda_storage::Store; +use ethlambda_types::ShortRoot; use ethlambda_types::beacon::builder::SignedBlindedBeaconBlock; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{HashTreeRoot as _, Root, Slot}; @@ -46,6 +47,18 @@ pub(crate) enum ClaimError { } impl ClaimError { + /// Logs the refusal, naming the root that holds the slot. + pub(crate) fn log(&self, slot: Slot) { + match self { + Self::Finalized => warn!(slot, "Refusing a block for a finalized slot"), + Self::Taken(root) => warn!( + slot, + existing = %ShortRoot(&root.0), + "Refusing a second, different block for the slot" + ), + } + } + /// The 400 a publish endpoint answers with. pub(crate) fn into_api_error(self) -> ApiError { match self { @@ -162,7 +175,7 @@ pub(crate) async fn post_blinded_block( } }; if let Err(err) = published.claim(slot, root, finalized_slot(&store)) { - warn!(slot, ?err, "Refusing a block for the slot"); + err.log(slot); return err.into_api_error().into_response(); } diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index a14f29e79..d6262dc37 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -256,7 +256,7 @@ async fn post_block( let root = signed_block.message.hash_tree_root(); let slot = signed_block.message.slot; if let Err(err) = published.claim(slot, root, finalized_slot(&store)) { - warn!(slot, ?err, "Refusing a block for the slot"); + err.log(slot); return err.into_api_error().into_response(); } let block = containers::SignedBeaconBlock::Fulu(signed_block); @@ -336,7 +336,7 @@ async fn post_gloas_block( let slot = block.slot(); let root = block.message_hash_tree_root(); if let Err(err) = published.claim(slot, root, finalized_slot(store)) { - warn!(slot, ?err, "Refusing a block for the slot"); + err.log(slot); return err.into_api_error().into_response(); } // A gloas block carries no blobs of its own: its data columns travel with @@ -621,12 +621,18 @@ async fn produce( fetch_bid(request.as_ref(), slot), ); let graffiti = graffiti::with_client_versions(graffiti, el_version.as_ref(), own_version); - let packed = pack(store, sync_pool, &state, head_root, slot, &config); let local = built.and_then(|built| local_parts(built, slot)); - let mut bid = request + let bid = request .as_ref() .zip(bid) .and_then(|(request, bid)| accepted_bid(&state, &config, request, bid)); + // Nothing to build a block from: skip the packing, whose sync aggregate + // costs a BLS verification. + let (local, mut bid) = match (local, bid) { + (Err(err), None) => return Err(err), + pair => pair, + }; + let packed = pack(store, sync_pool, &state, head_root, slot, &config); let local_offer = local.as_ref().ok().map(|parts| LocalOffer { value: wei_u128(&parts.built.block_value), @@ -1720,7 +1726,29 @@ mod tests { async fn without_a_bid_or_a_local_payload_there_is_no_block() { let world = builder_world(true, SyncStatus::Synced).await; let answer = produce_answer(&world, Some(dead_engine()), None, "").await; - assert_ne!(answer.status, StatusCode::OK, "{}", answer.json); + // The local error: the execution client did not start building. + assert_eq!( + answer.status, + StatusCode::SERVICE_UNAVAILABLE, + "{}", + answer.json + ); + assert_eq!(world.relay.header_requests().len(), 1); + } + + #[tokio::test] + async fn a_bid_whose_block_fails_to_build_falls_back_to_a_local_block_that_fails_too() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |_| {}); + let engine = + super::super::validator_client_tests::fake_execution_client_paying("0x2a", false).await; + // The infinity reveal without the skip flag fails the randao check in + // both blocks' state transitions. + let uri = produce_uri(world.slot, &infinity_reveal(), ""); + let request = Request::get(uri).body(Body::empty()).unwrap(); + let (status, json) = respond(world.app(Some(engine)), request).await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{json}"); + assert_eq!(json["message"], "the block failed to build"); assert_eq!(world.relay.header_requests().len(), 1); } @@ -1750,7 +1778,9 @@ mod tests { .record([®istration], compute_epoch_at_slot(world.slot)); world.builder = dead; let answer = produce(&world, "0x2a", false, "").await; - answer.assert(&world, false, "42", 0); + assert_eq!(answer.status, StatusCode::OK, "{}", answer.json); + assert_eq!(answer.blinded.as_deref(), Some("false"), "{}", answer.json); + assert_eq!(answer.value.as_deref(), Some("42"), "{}", answer.json); } #[tokio::test] @@ -1958,9 +1988,26 @@ mod tests { let blinded = signed_blinded(&world, 1, None); let (status, json) = post_blinded(app, &blinded).await; assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!( + json["message"], + "this node already published a different block for the slot" + ); assert!(world.relay.blinded_blocks().is_empty()); } + /// The blinded route takes the publish limit, not axum's 2 MiB default. + #[tokio::test] + async fn a_blinded_block_over_the_default_body_limit_is_not_a_413() { + let world = builder_world(true, SyncStatus::Synced).await; + let request = Request::post("/eth/v2/beacon/blinded_blocks") + .header("content-type", "application/octet-stream") + .header("eth-consensus-version", "fulu") + .body(Body::from(vec![0u8; 3 * 1024 * 1024])) + .unwrap(); + let (status, _) = respond(publish_app(&world), request).await; + assert_ne!(status, StatusCode::PAYLOAD_TOO_LARGE); + } + #[tokio::test] async fn an_oversized_blinded_block_is_a_413() { let world = builder_world(true, SyncStatus::Synced).await; diff --git a/docs/rpc.md b/docs/rpc.md index d7e4c0bb9..4772c3027 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -454,8 +454,8 @@ the chain actor writes, so no request waits on the actor. concurrently with the local build, bounded by its own timeout, and an error or a timeout means no bid. A bid is checked before it competes: a non-zero value, the head's execution block as its parent, the proposal's `prev_randao` - and timestamp, the parent's block number plus one, the withdrawals root, the blob commitment and blob gas bounds, - a gas limit within one step of the parent's toward the registered target, + and timestamp, the parent's block number plus one, the withdrawals root, + the blob commitment and blob gas bounds, a gas limit within one step of the parent's toward the registered target, and the builder's signature. Its fee recipient is not checked (see `spec_deviations.md`). A bid that passes goes through `assemble_blinded_block` with the packed attestations, operations and sync aggregate, so the blinded @@ -515,8 +515,8 @@ the chain actor writes, so no request waits on the actor. publishes the full block and its data columns; it returns over gossip and imports like any block, and nothing is gossiped here. The body limit is the one `beacon/blocks` has (`413` over it). The builder's answer is logged - from the submission task, an error when it did not take a signed block. The status is the - builder's: 202 becomes `200`, a builder `400` stays `400`, and anything else + from the submission task, an error when it did not take a signed block. + The status is the builder's: 202 becomes `200`, a builder `400` stays `400`, and anything else is a `500`. There is no local fallback, since a second block would be an equivocation. With no `--builder` the answer is `500`. - **The operation pool** holds, beside the attestation pool and in memory in From 3ade3a423ce5686cfe2bfdc901a287878931f8c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:48:19 -0300 Subject: [PATCH 13/20] docs: point at the fulu builder flow from the README and CLAUDE.md The README's beacon flag table, rewritten on main by #716, predates `--builder`, and CLAUDE.md described the builder containers and the blinded state root but not the node's flow that uses them. The retry arm's comment also differed from main's: main says the sync aggregate "was verified above", which stops holding here once `pack` does that check, and this PR kept the older wording. It now names `pack`, so the diff against main reads as the move it is. --- CLAUDE.md | 11 +++++++++++ README.md | 1 + crates/net/rpc/src/beacon/proposal.rs | 8 ++++---- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a775bae50..aa67c4bca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -588,6 +588,17 @@ default** on its two SSZ endpoints and bare integers in JSON, because `checkpoint_sync.rs` and other clients' lean sync read those bytes and may send no `Accept` at all; `/lean/v0/blocks/finalized` now answers JSON when asked for it by name. +With `--builder`, the beacon surface also runs the **fulu builder flow +(MEV-Boost)**. `register_validator` forwards registrations to the builder +(`beacon/registrations.rs`, client in `beacon/builder_client/`). +`produceBlockV3` asks for the builder's bid alongside the local build and +serves a blinded block when the bid passes its checks and wins on value +(`beacon/builder_flow/`: bid checks, circuit breaker, boost-factor choice). +`blinded_blocks` hands the signed block back for the builder to publish, with +no local fallback, since a second block for the slot would be an equivocation. Gloas +has no blinded blocks; its builder market is separate (the **Gloas** bullet +under Beacon Chain STF). + See [`docs/rpc.md`](docs/rpc.md) for the full reference: CLI flags and defaults, the lean API endpoints (health, finalized state/block, justified checkpoint, blocks by root/slot, fork-choice tree + D3.js UI, runtime aggregator toggle), the Beacon API endpoints and the three ids they refuse, the metrics/debug endpoints (Prometheus `/metrics`, jemalloc heap profiling), the Hive test-driver endpoints, plus request/response shapes, status codes, and content types. ## Beacon Chain types (`crates/common/types/src/beacon/`) diff --git a/README.md b/README.md index 116373eaf..d9cc433a6 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,7 @@ what they sign. | `--checkpoint-sync-url` | Where the anchor comes from. Required on a fresh data directory for a built-in network, since those never start from genesis. A resumable data directory is used before it; a network loaded from a directory anchors at its own `genesis.ssz` when no URL is given | | `--node-key` | Optional, but worth persisting: the columns this node custodies are a function of its node id, so without a key file it changes identity, and custody set, on every restart | | `--execution-endpoint`, `--execution-jwt-secret` | Optional Engine API pairing, given together. Without them, blocks import without payload validation | +| `--builder` | Optional builder API endpoint, normally MEV-Boost. With it, a fulu proposal from a validator registered through this node gets a blinded block when the builder's bid wins; without it every block is built locally | A built-in network needs nothing else on disk. Its `eth-clients` `config.yaml` and bootnode list, and the two genesis values the fork digest is computed from, diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index d6262dc37..c5d0e66f8 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -839,10 +839,10 @@ fn local_block( options, ) { Ok(block) => block, - // `pack_attestations` and `pack_operations` check every candidate - // against this state, and the sync aggregate is verified against it - // too, so this should not happen; but a block without them still earns - // the proposal, and one that fails to build earns nothing. + // `pack` checked every attestation and operation against this state + // and verified the sync aggregate against it, so this should not + // happen; but a block without them still earns the proposal, and one + // that fails to build earns nothing. Err(err) if packed.droppable() => { warn!(%slot, %err, "Block with attestations, operations or a sync aggregate failed to build; retrying without"); assemble_block( From 074b459bd745c3110fdfc85705d1390afd8431b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:20:37 -0300 Subject: [PATCH 14/20] fix(rpc): address the security review of the fulu builder flow - Refuse a block slot that has not started yet (gossip's `is_future_slot`: more than MAXIMUM_GOSSIP_CLOCK_DISPARITY, 500 ms, ahead of this node's clock) before any state is advanced to it, on produceBlockV3, produceBlockV4, both publishBlockV2 paths and blinded_blocks. Without it a caller naming a far-future slot had the node run every epoch transition up to it. produceBlockV4 is outside this PR's diff but had the same unbounded advance. A validator client asks once its slot has started, and a block further ahead would be ignored on gossip anyway. - Accept a bid that pays nothing: it no longer fails validation, and `choose` never lets it beat a local payload whatever the boost factor, so it is served only when the local payload or its block fails. The `zero_value` rejection label is gone. - Tell a builder's 2xx other than 202 from a refusal: a new `BuilderError::UnexpectedSuccess` (`unexpected_success` outcome label), its own error log and 500 message saying the builder may still publish the block. Document that only 202 is success and that `broadcast_validation` is not read on blinded_blocks. - Write out line breaks in a builder's error text as `\n` and `\r` before the 1 KiB cap, so it cannot forge a log line (`truncate_error_text` becomes `error_text`). This and the 202 change touch builder_client, which comes from the base PRs. - Log why advancing a state failed at all five advance sites instead of dropping the error. - Tests asked for in review: blinded-then-full on one slot, a claim on the gloas publish path, a refused blinded block leaving the slot unclaimed, two different blocks posted at once, a slot that has not started on every endpoint, and a builder answering 200. Tests that produced a block on a clock still in the slot before now pin the clock at the proposal slot. Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- .../state_transition/src/beacon/gossip/mod.rs | 2 +- crates/net/rpc/src/beacon/blinded_blocks.rs | 30 +- .../rpc/src/beacon/builder_client/error.rs | 14 +- .../net/rpc/src/beacon/builder_client/mod.rs | 20 +- .../rpc/src/beacon/builder_client/tests.rs | 49 ++- crates/net/rpc/src/beacon/builder_flow/bid.rs | 20 +- .../net/rpc/src/beacon/builder_flow/choice.rs | 26 ++ .../rpc/src/beacon/builder_market_tests.rs | 62 ++-- crates/net/rpc/src/beacon/gloas_proposal.rs | 14 +- crates/net/rpc/src/beacon/proposal.rs | 304 ++++++++++++++++-- .../rpc/src/beacon/validator_client_tests.rs | 69 +++- docs/metrics.md | 4 +- docs/rpc.md | 36 ++- 13 files changed, 555 insertions(+), 95 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 701fa705c..0151ce4b7 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -512,7 +512,7 @@ pub(crate) fn slot_start_ms(config: &Config, slot: Slot) -> u64 { /// The specification's `is_future_slot`: `slot` starts later than `now_ms` /// plus the gossip clock disparity allowance. -pub(crate) fn is_future_slot(config: &Config, slot: Slot, now_ms: u64) -> bool { +pub fn is_future_slot(config: &Config, slot: Slot, now_ms: u64) -> bool { slot_start_ms(config, slot) > now_ms.saturating_add(MAXIMUM_GOSSIP_CLOCK_DISPARITY) } diff --git a/crates/net/rpc/src/beacon/blinded_blocks.rs b/crates/net/rpc/src/beacon/blinded_blocks.rs index 97262ebd2..bc7800a7d 100644 --- a/crates/net/rpc/src/beacon/blinded_blocks.rs +++ b/crates/net/rpc/src/beacon/blinded_blocks.rs @@ -27,9 +27,14 @@ use tracing::{error, info, warn}; use super::builder_client::BuilderError; use super::builder_flow::Builder; -use super::proposal::require_fulu_slot; +use super::proposal::{refuse_future_slot, require_fulu_slot}; use super::{ApiError, BodyEncoding}; +/// The 500's message when the builder answers a 2xx other than 202: not a +/// refusal, since the builder may still publish the block. +pub(crate) const NOT_202: &str = + "the builder answered with a success other than 202 Accepted; it may still publish the block"; + /// The block root this node published or handed to the builder for each /// slot, so a second, different block for a slot is refused instead of /// broadcast. Pruned below the finalized slot. @@ -110,7 +115,10 @@ pub(crate) fn finalized_slot(store: &Store) -> Slot { /// committed, so a validator client that hangs up must not cancel the /// reveal. The answer is the builder's: 202 becomes 200 and a 400 is passed /// through. Anything else is a 500, with no local fallback, since a second -/// block would be an equivocation. +/// block would be an equivocation; another 2xx gets a 500 and a log line of +/// its own, since that builder may have taken the block. +/// `broadcast_validation` is not read: these checks always run, and the +/// builder, not this node, broadcasts. pub(crate) async fn post_blinded_block( State(store): State, builder: Option>>, @@ -141,7 +149,9 @@ pub(crate) async fn post_blinded_block( &store.config(), slot, "blinded blocks are accepted for fulu slots only", - ) { + ) + .and_then(|()| refuse_future_slot(&store.config(), slot)) + { return err.into_response(); } let Some(parent_state) = store.get_state(&signed.message.parent_root).ok().flatten() else { @@ -153,8 +163,10 @@ pub(crate) async fn post_blinded_block( if slot <= parent_state.slot() { return Err("the block is not after its parent"); } - let state = advance_to_slot(&parent_state, slot, &config) - .map_err(|_| "advancing the parent state failed")?; + let state = advance_to_slot(&parent_state, slot, &config).map_err(|err| { + warn!(slot, %err, "Advancing a published blinded block's parent state failed"); + "advancing the parent state failed" + })?; let proposer = get_beacon_proposer_index(&state).map_err(|_| "no proposer for the slot")?; if signed.message.proposer_index != proposer { return Err("wrong proposer for the slot"); @@ -191,6 +203,11 @@ pub(crate) async fn post_blinded_block( Err(BuilderError::BadRequest(message)) => { error!(slot, %message, "The builder refused the signed blinded block") } + Err(BuilderError::UnexpectedSuccess { status }) => error!( + slot, + status, + "The builder answered the signed blinded block with a success other than 202 Accepted, which submitBlindedBlockV2 requires; it may still publish the block, so check the chain before acting" + ), Err(err) => error!(slot, %err, "The builder did not take the signed blinded block"), } result @@ -201,6 +218,9 @@ pub(crate) async fn post_blinded_block( Ok(Err(BuilderError::BadRequest(message))) => { ApiError::BadRequestDetail(message).into_response() } + Ok(Err(BuilderError::UnexpectedSuccess { .. })) => { + ApiError::Internal(NOT_202).into_response() + } Ok(Err(_)) => ApiError::Internal("the builder did not take the block").into_response(), Err(err) => { error!(slot, %err, "Submitting the blinded block failed"); diff --git a/crates/net/rpc/src/beacon/builder_client/error.rs b/crates/net/rpc/src/beacon/builder_client/error.rs index 6da14b3c1..34e1d0786 100644 --- a/crates/net/rpc/src/beacon/builder_client/error.rs +++ b/crates/net/rpc/src/beacon/builder_client/error.rs @@ -4,7 +4,7 @@ use std::time::Duration; use reqwest::StatusCode; -use super::{read_body, truncate_error_text}; +use super::{error_text, read_body}; /// A failed builder API call. /// @@ -26,6 +26,13 @@ pub enum BuilderError { /// A status the call does not expect. #[error("builder answered {status}: {body}")] Status { status: u16, body: String }, + /// A success status other than the 202 `submitBlindedBlockV2` succeeds + /// with. Told apart from [`Status`](Self::Status) because the builder may + /// well have taken the block: 200 is what the V1 call answered. + #[error( + "builder answered {status}, a success but not the 202 Accepted submitBlindedBlockV2 requires; it may have taken the block anyway" + )] + UnexpectedSuccess { status: u16 }, /// A body that does not decode as what the call returns. #[error("builder response did not decode: {0}")] Decode(String), @@ -67,13 +74,13 @@ impl BuilderError { if status != StatusCode::BAD_REQUEST { return Self::Status { status: status.as_u16(), - body: truncate_error_text(body), + body: error_text(body), }; } let message = serde_json::from_str::(&body) .map(|error| error.message) .unwrap_or(body); - Self::BadRequest(truncate_error_text(message)) + Self::BadRequest(error_text(message)) } /// How the call ended, as a metric label. @@ -83,6 +90,7 @@ impl BuilderError { Self::Transport(_) => "unreachable", Self::BadRequest(_) => "bad_request", Self::Status { .. } => "unexpected_status", + Self::UnexpectedSuccess { .. } => "unexpected_success", Self::Decode(_) | Self::Incomplete(_) | Self::TooLarge { .. } => "invalid_response", Self::InvalidUrl(_) => "invalid_url", Self::UnsupportedFork(_) => "unsupported_fork", diff --git a/crates/net/rpc/src/beacon/builder_client/mod.rs b/crates/net/rpc/src/beacon/builder_client/mod.rs index d702e3d03..0b0fe0809 100644 --- a/crates/net/rpc/src/beacon/builder_client/mod.rs +++ b/crates/net/rpc/src/beacon/builder_client/mod.rs @@ -368,8 +368,18 @@ pub(super) async fn read_body( Ok(body) } -/// `text` cut to at most [`MAX_ERROR_TEXT`] bytes, at a char boundary. -pub(super) fn truncate_error_text(mut text: String) -> String { +/// `text` on one line, its line breaks written out as `\n` and `\r`, then cut +/// to at most [`MAX_ERROR_TEXT`] bytes at a char boundary. +/// +/// A builder's text reaches this node's logs and a validator client's error +/// message, where a raw line break would start what reads as a log line of +/// its own. +pub(super) fn error_text(text: String) -> String { + let mut text = if text.contains(['\n', '\r']) { + text.replace('\n', "\\n").replace('\r', "\\r") + } else { + text + }; if text.len() > MAX_ERROR_TEXT { let mut end = MAX_ERROR_TEXT; while !text.is_char_boundary(end) { @@ -413,10 +423,14 @@ fn outcome(result: &Result) -> &'static str { result.as_ref().map_or_else(BuilderError::label, |_| "ok") } -/// A submission's answer: only 202 is success. +/// A submission's answer: only 202 is success. Any other 2xx is an error of +/// its own, since the builder may have taken the block all the same. async fn submission_result(response: reqwest::Response) -> Result<(), BuilderError> { match response.status() { StatusCode::ACCEPTED => Ok(()), + status if status.is_success() => Err(BuilderError::UnexpectedSuccess { + status: status.as_u16(), + }), _ => Err(BuilderError::from_status(response, SUBMIT_TIMEOUT).await), } } diff --git a/crates/net/rpc/src/beacon/builder_client/tests.rs b/crates/net/rpc/src/beacon/builder_client/tests.rs index b2874c536..3321f0493 100644 --- a/crates/net/rpc/src/beacon/builder_client/tests.rs +++ b/crates/net/rpc/src/beacon/builder_client/tests.rs @@ -212,6 +212,26 @@ async fn a_long_error_is_cut_short() { assert_eq!(message.len(), 1024); } +/// Line breaks are written out, so a builder's text cannot start a log line of +/// its own, and the escaped text is what the length cap applies to. +#[tokio::test] +async fn line_breaks_in_an_error_are_written_out() { + let relay = MockRelay::start(GENESIS_FORK_VERSION).await; + let body = b"bad block\r\nINFO forged line".to_vec(); + relay.answer_headers_with(StatusCode::BAD_REQUEST, TEXT, body, None); + let Err(BuilderError::BadRequest(message)) = get_header(&client_for(&relay)).await else { + panic!("expected a bad request"); + }; + assert_eq!(message, r"bad block\r\nINFO forged line"); + + let relay = MockRelay::start(GENESIS_FORK_VERSION).await; + relay.answer_headers_with(StatusCode::BAD_REQUEST, TEXT, vec![b'\n'; 1000], None); + let Err(BuilderError::BadRequest(message)) = get_header(&client_for(&relay)).await else { + panic!("expected a bad request"); + }; + assert_eq!(message.len(), 1024); +} + #[tokio::test] async fn a_server_error_is_an_unexpected_status() { let relay = MockRelay::start(GENESIS_FORK_VERSION).await; @@ -384,13 +404,32 @@ async fn a_relay_that_refuses_a_block_says_why() { #[tokio::test] async fn only_202_is_a_successful_submission() { - for status in [StatusCode::OK, StatusCode::INTERNAL_SERVER_ERROR] { + let relay = MockRelay::start(GENESIS_FORK_VERSION).await; + relay.answer_submissions_with(StatusCode::INTERNAL_SERVER_ERROR); + assert!(matches!( + client_for(&relay) + .submit_blinded_block(&blinded_block(), ForkName::Fulu) + .await, + Err(BuilderError::Status { status: 500, .. }) + )); +} + +/// Another 2xx is still a failure, but one of its own, labelled apart from a +/// refusal or a server error. +#[tokio::test] +async fn a_2xx_other_than_202_is_an_unexpected_success() { + for status in [StatusCode::OK, StatusCode::NO_CONTENT] { let relay = MockRelay::start(GENESIS_FORK_VERSION).await; relay.answer_submissions_with(status); - assert!(matches!( - client_for(&relay).submit_blinded_block(&blinded_block(), ForkName::Fulu).await, - Err(BuilderError::Status { status: answered, .. }) if answered == status.as_u16() - )); + let result = client_for(&relay) + .submit_blinded_block(&blinded_block(), ForkName::Fulu) + .await; + let Err(err @ BuilderError::UnexpectedSuccess { status: answered }) = result else { + panic!("{status}: expected an unexpected success, got {result:?}"); + }; + assert_eq!(answered, status.as_u16()); + assert_eq!(err.label(), "unexpected_success"); + assert!(err.to_string().contains("202 Accepted"), "{err}"); } } diff --git a/crates/net/rpc/src/beacon/builder_flow/bid.rs b/crates/net/rpc/src/beacon/builder_flow/bid.rs index 88686a98f..48f2b8366 100644 --- a/crates/net/rpc/src/beacon/builder_flow/bid.rs +++ b/crates/net/rpc/src/beacon/builder_flow/bid.rs @@ -17,8 +17,6 @@ use ethlambda_types::beacon::containers::capella; use ethlambda_types::beacon::primitives::{HashTreeRoot as _, Version}; use ethlambda_types::beacon::signing::{compute_builder_domain, compute_signing_root}; -use crate::beacon::bid_selection::wei_u128; - /// EIP-4844's blob gas per blob. pub(crate) const GAS_PER_BLOB: u64 = 1 << 17; @@ -39,7 +37,6 @@ pub(crate) struct BidContext<'a> { /// Why a bid was refused. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Rejection { - ZeroValue, ParentHash, PrevRandao, Timestamp, @@ -55,7 +52,6 @@ impl Rejection { /// The metric label. pub(crate) fn label(self) -> &'static str { match self { - Self::ZeroValue => "zero_value", Self::ParentHash => "parent_hash", Self::PrevRandao => "prev_randao", Self::Timestamp => "timestamp", @@ -74,13 +70,12 @@ impl Rejection { /// /// The bid's fork label is not checked here: an electra bid at a fulu slot is /// accepted on purpose (the containers are identical, so a lagging relay's bid -/// stays usable), and `BuilderClient` already refuses every other fork. +/// stays usable), and `BuilderClient` already refuses every other fork. Nor is +/// its value: a bid that pays nothing still stands in for a local block that +/// failed, and `choice::choose` keeps it from beating one that did not. pub(crate) fn validate(bid: &SignedBuilderBid, context: &BidContext) -> Result<(), Rejection> { let message = &bid.message; let header = &message.header; - if wei_u128(&message.value) == 0 { - return Err(Rejection::ZeroValue); - } if header.parent_hash != context.inputs.parent_hash { return Err(Rejection::ParentHash); } @@ -207,11 +202,16 @@ mod tests { assert_eq!(rejection(&bid(|_| {})), None); } + /// What a bid pays is for `choice::choose` to weigh, not a check here. + #[test] + fn a_bid_that_pays_nothing_is_accepted() { + assert_eq!(rejection(&bid(|bid| bid.value = Uint256::from(0u64))), None); + } + #[test] fn each_check_refuses_its_own_field() { type Edit = fn(&mut BuilderBid); - let cases: [(Rejection, Edit); 9] = [ - (Rejection::ZeroValue, |bid| bid.value = Uint256::from(0u64)), + let cases: [(Rejection, Edit); 8] = [ (Rejection::ParentHash, |bid| { bid.header.parent_hash = ExecutionBlockHash::repeat_byte(9) }), diff --git a/crates/net/rpc/src/beacon/builder_flow/choice.rs b/crates/net/rpc/src/beacon/builder_flow/choice.rs index fd5712b10..2c403faae 100644 --- a/crates/net/rpc/src/beacon/builder_flow/choice.rs +++ b/crates/net/rpc/src/beacon/builder_flow/choice.rs @@ -37,6 +37,10 @@ pub(crate) struct LocalOffer { /// A factor of `u64::MAX` always takes the builder, except that the execution /// client's `shouldOverrideBuilder` still wins. When one side failed, the /// other stands. +/// +/// A bid that pays nothing never beats a local payload, whatever the factor: +/// it stands only in for one that failed, here or when its block fails to +/// assemble. pub(crate) fn choose( local: Option, bid_value: Option, @@ -46,6 +50,7 @@ pub(crate) fn choose( (None, None) => None, (Some(_), None) => Some(Source::Local), (None, Some(_)) => Some(Source::Builder), + (Some(_), Some(0)) => Some(Source::Local), (Some(local), Some(_)) if local.should_override_builder => Some(Source::Local), (Some(_), Some(_)) if boost_factor == u64::MAX => Some(Source::Builder), (Some(local), Some(bid)) => { @@ -138,6 +143,27 @@ mod tests { u64::MAX, Some(Source::Local), ), + ( + "a bid paying nothing loses to local", + local(0), + Some(0), + 100, + Some(Source::Local), + ), + ( + "even under u64::MAX", + local(1), + Some(0), + u64::MAX, + Some(Source::Local), + ), + ( + "a bid paying nothing stands in for no local", + None, + Some(0), + 0, + Some(Source::Builder), + ), ]; for (name, local, bid, factor, expected) in cases { assert_eq!(choose(local, bid, factor), expected, "{name}"); diff --git a/crates/net/rpc/src/beacon/builder_market_tests.rs b/crates/net/rpc/src/beacon/builder_market_tests.rs index 4f89fd600..a14e7e930 100644 --- a/crates/net/rpc/src/beacon/builder_market_tests.rs +++ b/crates/net/rpc/src/beacon/builder_market_tests.rs @@ -65,7 +65,8 @@ use crate::{ /// The slot the block is built for. The head block sits one slot earlier, and /// the clock is at the head's slot, so this one is the next slot: the one bids -/// and preferences are accepted for. +/// and preferences are accepted for. Production runs once this slot has +/// started, on [`World::proposing`]'s clock. const SLOT: u64 = 33; const HEAD_SLOT: u64 = SLOT - 1; const GWEI: u64 = 1_000_000_000; @@ -225,8 +226,20 @@ struct World { } impl World { - /// The head block at slot 32 on a clock where that slot is running. + /// The head block at slot 32 on a clock where that slot is running: the + /// moment bids and preferences for the next slot arrive. fn new() -> Self { + Self::at_clock(HEAD_SLOT) + } + + /// [`World::new`] on a clock where [`SLOT`] is running: the moment a + /// validator client asks for its block, since production refuses a slot + /// that has not started yet. + fn proposing() -> Self { + Self::at_clock(SLOT) + } + + fn at_clock(clock_slot: u64) -> Self { let state = chain_state(); let genesis = genesis_block(); let genesis_root = genesis.message_hash_tree_root(); @@ -245,7 +258,7 @@ impl World { chain_config(), block, head_root, - HEAD_SLOT, + clock_slot, ); store.insert_signed_block(genesis_root, genesis).unwrap(); store.insert_state(genesis_root, state.clone()).unwrap(); @@ -484,7 +497,7 @@ fn self_built(reply: &Reply) -> bool { #[tokio::test] async fn a_bid_worth_more_than_the_local_payload_is_built_on_and_returned_bare() { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(GWEI, false).await; let bid = world.bid(5, address(0xcc)); world.prime_bid(&bid); @@ -516,7 +529,7 @@ async fn a_bid_worth_more_than_the_local_payload_is_built_on_and_returned_bare() #[tokio::test] async fn a_bid_won_block_is_served_as_ssz_too() { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(GWEI, false).await; let bid = world.bid(5, address(0xcc)); world.prime_bid(&bid); @@ -535,7 +548,7 @@ async fn a_bid_won_block_is_served_as_ssz_too() { #[tokio::test] async fn nothing_is_cached_for_a_bid_block_and_a_self_build_envelope_for_it_is_refused() { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(GWEI, false).await; world.prime_bid(&world.bid(5, address(0xcc))); let app = world.app(Some(&engine)); @@ -609,7 +622,7 @@ async fn the_local_payload_wins_ties_a_zero_factor_a_floor_and_an_override() { (1, true, 0, u64::MAX), ]; for (local_wei, should_override, min_bid, factor) in cases { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(local_wei, should_override).await; world.prime_bid(&world.bid(5, address(0xcc))); @@ -644,7 +657,7 @@ async fn the_local_payload_wins_ties_a_zero_factor_a_floor_and_an_override() { async fn a_bid_wins_when_the_local_build_is_below_it_by_one_unit_of_weight() { // 1e9 wei is 1 gwei: a tie at factor 100. One wei less and the bid wins. for (local_wei, bid_wins) in [(GWEI, false), (GWEI - 1, true)] { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(local_wei, false).await; world.prime_bid(&world.bid(1, address(0xcc))); @@ -657,7 +670,7 @@ async fn a_bid_wins_when_the_local_build_is_below_it_by_one_unit_of_weight() { #[tokio::test] async fn without_an_engine_a_viable_bid_is_built_on_and_none_is_a_503() { - let world = World::new(); + let world = World::proposing(); let reply = produce(&world, None, true, &builder_config(0, 0)).await; assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE); @@ -675,7 +688,7 @@ async fn without_an_engine_a_viable_bid_is_built_on_and_none_is_a_503() { #[tokio::test] async fn a_failed_local_build_falls_back_to_a_viable_bid() { - let world = World::new(); + let world = World::proposing(); let engine = dead_engine(); let reply = produce(&world, Some(&engine), true, &builder_config(0, 0)).await; assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE); @@ -701,7 +714,7 @@ async fn bids_that_do_not_fit_the_slot_are_left_to_the_local_payload() { }), ]; for (name, change) in mismatches { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(1, false).await; let mut bid = world.bid(5, address(0xcc)); change(&mut bid.message); @@ -718,7 +731,7 @@ async fn bids_that_do_not_fit_the_slot_are_left_to_the_local_payload() { #[tokio::test] async fn a_bid_paying_someone_other_than_the_preferred_recipient_is_skipped() { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(1, false).await; world.prime_preferences(address(0xaa), 30_000_000); world.prime_bid(&world.bid(5, address(0xcc))); @@ -731,7 +744,7 @@ async fn a_bid_paying_someone_other_than_the_preferred_recipient_is_skipped() { #[tokio::test] async fn a_bid_paying_the_preferred_recipient_is_taken() { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(1, false).await; world.prime_preferences(address(0xaa), 30_000_000); let bid = world.bid(5, address(0xaa)); @@ -745,7 +758,7 @@ async fn a_bid_paying_the_preferred_recipient_is_taken() { #[tokio::test] async fn the_self_build_takes_its_fee_recipient_and_gas_target_from_the_preferences() { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(GWEI, false).await; let proposer = world.prepared().proposer; world @@ -775,7 +788,7 @@ async fn the_self_build_takes_its_fee_recipient_and_gas_target_from_the_preferen #[tokio::test] async fn without_preferences_the_self_build_falls_back_to_the_prepared_recipient_and_parent_gas() { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(GWEI, false).await; let proposer = world.prepared().proposer; world @@ -799,7 +812,7 @@ async fn without_preferences_the_self_build_falls_back_to_the_prepared_recipient ); // Neither: the zero address. - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(GWEI, false).await; produce(&world, Some(&engine), true, &builder_config(0, 0)).await; assert_eq!( @@ -810,7 +823,7 @@ async fn without_preferences_the_self_build_falls_back_to_the_prepared_recipient #[tokio::test] async fn preferences_of_another_validator_are_not_the_proposers() { - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(GWEI, false).await; let proposer = world.prepared().proposer; let other = (proposer + 1) % 64; @@ -832,7 +845,7 @@ async fn preferences_of_another_validator_are_not_the_proposers() { #[tokio::test] async fn the_existing_error_paths_are_unchanged() { - let world = World::new(); + let world = World::proposing(); let app = world.app(None); // A pre-existing client sends the empty local-preferred config. let reply = send( @@ -846,6 +859,17 @@ async fn the_existing_error_paths_are_unchanged() { assert_eq!(reply.json()["message"], "the body is not a BuilderConfig"); } +/// A validator client asks once its slot has started. Ahead of that, here a +/// whole slot early, the slot is refused before the head state is advanced +/// to it, and before the missing execution client is noticed. +#[tokio::test] +async fn a_slot_that_has_not_started_is_refused() { + let world = World::new(); + let reply = produce(&world, None, false, &builder_config(0, 100)).await; + assert_eq!(reply.status, StatusCode::BAD_REQUEST); + assert_eq!(reply.json()["message"], "the slot has not started yet"); +} + // --------------------------------------------------------------------------- // POST execution_payload_bids // --------------------------------------------------------------------------- @@ -1268,7 +1292,7 @@ async fn the_new_routes_are_wired_with_the_production_layers() { async fn the_infinity_reveal_is_produced_only_when_the_flag_skips_its_verification() { use ethlambda_state_transition::beacon::bls::G2_POINT_AT_INFINITY; - let world = World::new(); + let world = World::proposing(); let engine = fake_engine(GWEI, false).await; let app = world.app(Some(&engine)); let infinity = BlsSignature(G2_POINT_AT_INFINITY); diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs index 51b57d76e..2eaf9467f 100644 --- a/crates/net/rpc/src/beacon/gloas_proposal.rs +++ b/crates/net/rpc/src/beacon/gloas_proposal.rs @@ -109,7 +109,9 @@ use crate::beacon::{ bid_selection::{LocalCandidate, PayloadChoice, bid_total_gwei, choose_payload, wei_u128}, builder_config::{BuilderConfig, decode_builder_config}, graffiti::{self, OwnVersion}, - proposal::{Blobs, CellKzgProofs, decimal, randao_verification, require_gloas_slot}, + proposal::{ + Blobs, CellKzgProofs, decimal, randao_verification, refuse_future_slot, require_gloas_slot, + }, validator::{FeeRecipients, head}, }; use crate::shared::content::{Encoding, ssz_response, with_consensus_version}; @@ -257,7 +259,9 @@ async fn post_produce_block( &store.config(), slot, "this endpoint serves gloas slots only", - ) { + ) + .and_then(|()| refuse_future_slot(&store.config(), slot)) + { return err.into_response(); } let builder_config = match decode_builder_config(&headers, &body) { @@ -679,8 +683,10 @@ fn prepare( ExecutionRequests::default() }; - let state = advance_to_slot(&head_state, slot, &config) - .map_err(|_| ApiError::Internal("advancing the head state failed"))?; + let state = advance_to_slot(&head_state, slot, &config).map_err(|err| { + warn!(%slot, %err, "Advancing the head state to the proposal slot failed"); + ApiError::Internal("advancing the head state failed") + })?; let proposer = get_beacon_proposer_index(&state) .map_err(|_| ApiError::Internal("no proposer for the slot"))?; if randao == RandaoVerification::Verify { diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index c5d0e66f8..39816dfb0 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -41,6 +41,7 @@ use ethlambda_state_transition::beacon::{ }, bls, data_columns::{self, SidecarError}, + gossip::is_future_slot, helpers::accessors::get_beacon_proposer_index, stf::{ block::{BlockProcessingOptions, RandaoVerification}, @@ -74,6 +75,7 @@ use libssz_derive::{SszDecode, SszEncode}; use libssz_types::SszList; use serde::Deserialize; use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; use tracing::{debug, error, info, warn}; use crate::beacon::bid_selection::wei_u128; @@ -178,7 +180,9 @@ async fn post_block( &store.config(), contents.signed_block.message.slot, "blocks are accepted for fulu slots only", - ) { + ) + .and_then(|()| refuse_future_slot(&store.config(), contents.signed_block.message.slot)) + { return err.into_response(); } @@ -201,8 +205,10 @@ async fn post_block( if slot <= parent_state.slot() { return Err(("the block is not after its parent", None)); } - let state = advance_to_slot(&parent_state, slot, &config) - .map_err(|_| ("advancing the parent state failed", None))?; + let state = advance_to_slot(&parent_state, slot, &config).map_err(|err| { + warn!(slot, %err, "Advancing a published block's parent state failed"); + ("advancing the parent state failed", None) + })?; let proposer = get_beacon_proposer_index(&state).map_err(|_| ("no proposer for the slot", None))?; if signed_block.message.proposer_index != proposer { @@ -295,7 +301,9 @@ async fn post_gloas_block( &config, block.slot(), "blocks are accepted for gloas slots only", - ) { + ) + .and_then(|()| refuse_future_slot(&config, block.slot())) + { return err.into_response(); } let parent_state = match store.get_state(&block.parent_root()) { @@ -309,8 +317,11 @@ async fn post_gloas_block( let verdict = { let block = block.clone(); tokio::task::spawn_blocking(move || { - let state = advance_to_slot(&parent_state, block.slot(), &config) - .map_err(|_| ApiError::Internal("advancing the parent state failed"))?; + let state = advance_to_slot(&parent_state, block.slot(), &config).map_err(|err| { + let slot = block.slot(); + warn!(slot, %err, "Advancing a published block's parent state failed"); + ApiError::Internal("advancing the parent state failed") + })?; let expected = ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index( &state, @@ -432,6 +443,26 @@ pub(super) fn require_fulu_slot( } } +/// Refuses a slot that has not started yet by this node's clock, before any +/// state is advanced to it. Advancing runs every slot and epoch transition up +/// to the slot, so with no bound a caller naming a far-future slot would hold +/// a thread for as long as those take. +/// +/// The bound is gossip's `is_future_slot`: the slot may start at most +/// `MAXIMUM_GOSSIP_CLOCK_DISPARITY` (500 ms) after now. A validator client asks +/// for its block and publishes it once the slot has started, so one on a +/// correct clock never meets the bound, and a block for a slot further ahead +/// would be ignored by every peer's gossip validation anyway. +pub(super) fn refuse_future_slot(config: &Config, slot: Slot) -> Result<(), ApiError> { + let now_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_millis() as u64); + if is_future_slot(config, slot, now_ms) { + return Err(ApiError::BadRequest("the slot has not started yet")); + } + Ok(()) +} + #[allow(clippy::too_many_arguments)] async fn get_block( Path(slot): Path, @@ -462,7 +493,9 @@ async fn get_block( &store.config(), slot, "block production is served for fulu only", - ) { + ) + .and_then(|()| refuse_future_slot(&store.config(), slot)) + { return err.into_response(); } let Some(engine) = engine else { @@ -602,8 +635,10 @@ async fn produce( if slot <= head_state.slot() { return Err(ApiError::BadRequest("slot is not after the head block")); } - let state = advance_to_slot(&head_state, slot, &config) - .map_err(|_| ApiError::Internal("advancing the head state failed"))?; + let state = advance_to_slot(&head_state, slot, &config).map_err(|err| { + warn!(%slot, %err, "Advancing the head state to the proposal slot failed"); + ApiError::Internal("advancing the head state failed") + })?; let fork = state.fork_name(); let proposer = ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index(&state) @@ -1272,6 +1307,66 @@ mod tests { assert_eq!(json["message"], "blocks are accepted for fulu slots only"); } + /// Two slots past the wall clock: production and both publish paths + /// refuse it before any state is advanced, and before the missing + /// execution client or the unknown parent is noticed. + #[tokio::test] + async fn a_slot_that_has_not_started_is_refused_before_advancing() { + let (store, _) = fulu_store(); + let future = crate::beacon::node::wall_slot(&store) + 2; + + let uri = produce_uri(future, &infinity_reveal(), "&skip_randao_verification"); + let request = Request::get(uri).body(Body::empty()).unwrap(); + let (status, json) = respond(produce_app(store.clone(), None), request).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], "the slot has not started yet"); + + let contents = FuluSignedBlockContents { + signed_block: SignedBeaconBlock { + message: BeaconBlock { + slot: future, + proposer_index: 0, + parent_root: H256::ZERO, + state_root: H256::ZERO, + body: containers::electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }, + kzg_proofs: Default::default(), + blobs: Default::default(), + }; + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let app = routes().with_state(store).layer(Extension(p2p)); + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "fulu") + .header(header::CONTENT_TYPE, crate::SSZ_CONTENT_TYPE) + .body(Body::from(contents.to_ssz())) + .unwrap(); + let (status, json) = respond(app, request).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], "the slot has not started yet"); + + let (store, _) = gloas_scheduled_store(); + let future = crate::beacon::node::wall_slot(&store) + 2; + let containers::SignedBeaconBlock::Gloas(block) = + crate::test_utils::gloas_beacon_block(future, H256::ZERO, H256::ZERO, H256::ZERO) + else { + unreachable!("built as gloas") + }; + let p2p: RpcToP2PRef = network.clone(); + let app = routes().with_state(store).layer(Extension(p2p)); + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "gloas") + .header(header::CONTENT_TYPE, crate::SSZ_CONTENT_TYPE) + .body(Body::from(block.to_ssz())) + .unwrap(); + let (status, json) = respond(app, request).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], "the slot has not started yet"); + assert!(network.blocks.lock().unwrap().is_empty()); + } + /// A fulu body in either encoding reaches the same check and gets the same /// answer, here the refusal of a slot the schedule does not place at fulu. #[tokio::test] @@ -1414,6 +1509,52 @@ mod tests { assert!(rejected.blocks.lock().unwrap().is_empty()); } + /// The gloas path claims the slot as the fulu ones do: the same block + /// again is a retry, and a different one the proposer signed is refused. + #[tokio::test] + async fn a_gloas_block_claims_its_slot() { + use ethlambda_state_transition::beacon::gloas_block_production::test_support::{ + config, parent_state, produce, sign_block, state_to_build_on, + }; + let parent = parent_state(); + let state = state_to_build_on(); + let produced = produce(&state, true, Vec::new()).unwrap(); + let (mut store, _anchor) = beacon_store_with_config(parent.clone(), config()); + store + .insert_state(produced.block.parent_root, parent) + .unwrap(); + let sign = |block: &containers::gloas::BeaconBlock| containers::gloas::SignedBeaconBlock { + signature: sign_block(&state, block), + message: block.clone(), + }; + let first = sign(&produced.block); + let mut other = produced.block.clone(); + other.body.graffiti = Bytes32::repeat_byte(0x42); + let second = sign(&other); + + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let app = routes().with_state(store).layer(Extension(p2p)); + let post = |signed: &containers::gloas::SignedBeaconBlock| { + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "gloas") + .header(header::CONTENT_TYPE, crate::SSZ_CONTENT_TYPE) + .body(Body::from(signed.to_ssz())) + .unwrap(); + respond(app.clone(), request) + }; + + assert_eq!(post(&first).await.0, StatusCode::OK); + assert_eq!(post(&first).await.0, StatusCode::OK); + let (status, json) = post(&second).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!( + json["message"], + "this node already published a different block for the slot" + ); + assert_eq!(network.blocks.lock().unwrap().len(), 2); + } + use crate::beacon::builder_client::mock_relay::MockRelay; use crate::beacon::builder_client::{BuilderClient, VersionedBid}; use crate::beacon::builder_flow::breaker::FallbackThresholds; @@ -1722,6 +1863,23 @@ mod tests { answer.assert(&world, true, Ð.to_string(), 1); } + #[tokio::test] + async fn a_bid_paying_nothing_stands_in_for_a_failed_local_payload() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(0, |_| {}); + let answer = produce_answer(&world, Some(dead_engine()), None, "").await; + answer.assert(&world, true, "0", 1); + } + + #[tokio::test] + async fn a_bid_paying_nothing_never_beats_a_local_payload() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(0, |_| {}); + let query = format!("&builder_boost_factor={}", u64::MAX); + let answer = produce(&world, "0x0", false, &query).await; + answer.assert(&world, false, "0", 1); + } + #[tokio::test] async fn without_a_bid_or_a_local_payload_there_is_no_block() { let world = builder_world(true, SyncStatus::Synced).await; @@ -1946,12 +2104,11 @@ mod tests { assert_eq!(world.relay.blinded_blocks(), [first.clone(), first]); } - #[tokio::test] - async fn a_block_published_in_full_blocks_a_different_blinded_one() { - let world = builder_world(true, SyncStatus::Synced).await; - // A full block for the slot from the local path, signed by its proposer. - let answer = produce(&world, "0x2a", false, "").await; - answer.assert(&world, false, "42", 1); + /// A full block for `world`'s slot from the local path, signed by its + /// proposer. + async fn signed_full_block(world: &BuilderWorld) -> FuluSignedBlockContents { + let answer = produce(world, "0x2a", false, "").await; + answer.assert(world, false, "42", 1); let mut contents: FuluBlockContents = serde_json::from_value(answer.json["data"].clone()).unwrap(); // The test store keys its head by a root that is not its header's, so @@ -1961,40 +2118,141 @@ mod tests { let proposer = get_beacon_proposer_index(&world.state).unwrap(); let domain = get_domain(&world.state, DOMAIN_BEACON_PROPOSER, None); let signing_root = compute_signing_root(contents.block.hash_tree_root(), domain); - let signed = FuluSignedBlockContents { + FuluSignedBlockContents { signed_block: electra::SignedBeaconBlock { message: contents.block, signature: sign_for(proposer as usize, signing_root), }, kzg_proofs: contents.kzg_proofs, blobs: contents.blobs, - }; + } + } - let network = Arc::new(RecordingNetwork::default()); + /// Both publish routes over one guard, gossiping into `network`. + fn publish_app_into(world: &BuilderWorld, network: &Arc) -> Router { let p2p: RpcToP2PRef = network.clone(); - let app = routes() + routes() .with_state(world.store.clone()) .layer(Extension(p2p)) - .layer(Extension(world.builder.clone())); + .layer(Extension(world.builder.clone())) + } + + async fn post_full( + app: Router, + signed: &FuluSignedBlockContents, + ) -> (StatusCode, serde_json::Value) { let request = Request::post("/eth/v2/beacon/blocks") .header("content-type", "application/octet-stream") .header("eth-consensus-version", "fulu") .body(Body::from(signed.to_ssz())) .unwrap(); - let (status, json) = respond(app.clone(), request).await; + respond(app, request).await + } + + const TAKEN: &str = "this node already published a different block for the slot"; + + #[tokio::test] + async fn a_block_published_in_full_blocks_a_different_blinded_one() { + let world = builder_world(true, SyncStatus::Synced).await; + let signed = signed_full_block(&world).await; + let network = Arc::new(RecordingNetwork::default()); + let app = publish_app_into(&world, &network); + + let (status, json) = post_full(app.clone(), &signed).await; assert_eq!(status, StatusCode::OK, "{json}"); assert_eq!(network.blocks.lock().unwrap().len(), 1); let blinded = signed_blinded(&world, 1, None); let (status, json) = post_blinded(app, &blinded).await; assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], TAKEN); + assert!(world.relay.blinded_blocks().is_empty()); + } + + #[tokio::test] + async fn a_blinded_block_handed_to_the_builder_blocks_a_different_full_one() { + let world = builder_world(true, SyncStatus::Synced).await; + let signed = signed_full_block(&world).await; + let network = Arc::new(RecordingNetwork::default()); + let app = publish_app_into(&world, &network); + + let blinded = signed_blinded(&world, 1, None); + let (status, json) = post_blinded(app.clone(), &blinded).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(world.relay.blinded_blocks(), [blinded]); + + let (status, json) = post_full(app, &signed).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], TAKEN); + assert!(network.blocks.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_refused_blinded_block_leaves_the_slot_unclaimed() { + let world = builder_world(true, SyncStatus::Synced).await; + let app = publish_app(&world); + let proposer = get_beacon_proposer_index(&world.state).unwrap() as usize; + let forged = signed_blinded(&world, 1, Some(proposer + 1)); assert_eq!( - json["message"], - "this node already published a different block for the slot" + post_blinded(app.clone(), &forged).await.0, + StatusCode::BAD_REQUEST + ); + + let genuine = signed_blinded(&world, 2, None); + let (status, json) = post_blinded(app, &genuine).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(world.relay.blinded_blocks(), [genuine]); + } + + /// Two different blocks for the slot in flight at once: the claim is + /// checked and taken under one lock, so exactly one reaches the builder. + #[tokio::test] + async fn of_two_different_blocks_posted_at_once_one_is_taken() { + let world = builder_world(true, SyncStatus::Synced).await; + let app = publish_app(&world); + let first = signed_blinded(&world, 1, None); + let second = signed_blinded(&world, 2, None); + + let ((first_status, _), (second_status, _)) = tokio::join!( + post_blinded(app.clone(), &first), + post_blinded(app, &second) ); + + let mut statuses = [first_status, second_status]; + statuses.sort(); + assert_eq!(statuses, [StatusCode::OK, StatusCode::BAD_REQUEST]); + let taken = if first_status == StatusCode::OK { + first + } else { + second + }; + assert_eq!(world.relay.blinded_blocks(), [taken]); + } + + #[tokio::test] + async fn a_blinded_block_for_a_slot_that_has_not_started_is_refused() { + let world = builder_world(true, SyncStatus::Synced).await; + let mut block = signed_blinded(&world, 1, None); + block.message.slot = crate::beacon::node::wall_slot(&world.store) + 2; + let (status, json) = post_blinded(publish_app(&world), &block).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], "the slot has not started yet"); assert!(world.relay.blinded_blocks().is_empty()); } + /// A 200 is success for V1 but not for `submitBlindedBlockV2`: a 500, with + /// a message saying the builder may have taken the block anyway. + #[tokio::test] + async fn a_builder_answering_200_instead_of_202_is_a_server_error_of_its_own() { + let world = builder_world(true, SyncStatus::Synced).await; + world.relay.answer_submissions_with(StatusCode::OK); + let block = signed_blinded(&world, 1, None); + let (status, json) = post_blinded(publish_app(&world), &block).await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{json}"); + assert_eq!(json["message"], blinded_blocks::NOT_202); + assert_eq!(world.relay.blinded_blocks(), [block]); + } + /// The blinded route takes the publish limit, not axum's 2 MiB default. #[tokio::test] async fn a_blinded_block_over_the_default_body_limit_is_not_a_413() { diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 654fb6d5d..aa1a665b8 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -119,7 +119,11 @@ async fn serve_at( fulu.current_sync_committee = sync_committee.clone(); fulu.next_sync_committee = sync_committee; - let (mut store, _root) = beacon_store_at(state.clone()); + // On a clock where the slot after the head is running, the slot these + // tests propose for: production refuses a slot that has not started, and + // the wall clock may still be in the epoch's first slot, the head's own. + let (mut store, _root) = + crate::test_utils::beacon_store_at_clock(state.clone(), state.slot() + 1); // A block built on this head names the head block's real root as its // parent, which the fixture's placeholder anchor block does not have: // keep the state under that root too, as a real anchor would be. @@ -139,8 +143,25 @@ async fn serve_at( /// ephemeral port, and connect this crate's client to it. Also hands back the /// payload attestation pool the router fills and the socket it listens on. async fn spawn_server( + store: Store, + engine: Option, +) -> ( + HttpBeaconNode, + Arc, + SharedPayloadAttestationPool, + std::net::SocketAddr, +) { + spawn_server_sharing(store, engine, SharedSyncCommitteePool::default()).await +} + +/// [`spawn_server`] pooling sync committee messages in `sync_pool`, so a second +/// node over the same head, on a clock a slot later, packs into its block what +/// the first one pooled: block production refuses a slot that has not started, +/// and a sync committee message is accepted only during its own slot. +async fn spawn_server_sharing( mut store: Store, engine: Option, + sync_pool: SharedSyncCommitteePool, ) -> ( HttpBeaconNode, Arc, @@ -160,7 +181,7 @@ async fn spawn_server( let router = crate::build_beacon_api_router(store, "ethlambda/test", "peer".into()) .layer(Extension(SyncStatusController::new(SyncStatus::Synced))) .layer(Extension(p2p)) - .layer(Extension(SharedSyncCommitteePool::default())) + .layer(Extension(sync_pool)) .layer(Extension(payload_pool.clone())) .layer(Extension( ethlambda_state_transition::beacon::builder_market::SharedBuilderMarket::default(), @@ -812,7 +833,9 @@ async fn the_json_block_carries_its_blobs_and_cell_proofs_as_hex() { // --------------------------------------------------------------------------- /// A fulu head state at the first slot of epoch 2, its header's root the one -/// the store's head is under, served on a clock pinned to that slot. +/// the store's head is under, served on a clock pinned to that slot, or +/// `clock_ahead` slots after it, pooling sync committee messages in +/// `sync_pool`. /// /// Unlike [`serve`] the head root is the root a block built on this state /// names as its parent, which is the root `process_sync_aggregate` reads back @@ -822,6 +845,8 @@ async fn the_json_block_carries_its_blobs_and_cell_proofs_as_hex() { /// own `fork` and the client's schedule give one domain. async fn serve_fulu_pinned( engine: Option, + sync_pool: SharedSyncCommitteePool, + clock_ahead: u64, ) -> ( Arc, BeaconState, @@ -868,9 +893,9 @@ async fn serve_fulu_pinned( config, block, head_root, - slot, + slot + clock_ahead, ); - let (client, network, _, _) = spawn_server(store, engine).await; + let (client, network, _, _) = spawn_server_sharing(store, engine, sync_pool).await; let genesis = client.genesis().await.expect("genesis"); let spec = client .spec() @@ -1039,12 +1064,17 @@ async fn the_validator_client_can_serve_on_the_sync_committee() { containers::SignedBeaconBlock, }; + let engine = fake_execution_client(0).await; + let sync_pool = SharedSyncCommitteePool::default(); let (client, state, network, context) = - serve_fulu_pinned(Some(fake_execution_client(0).await)).await; + serve_fulu_pinned(Some(engine.clone()), sync_pool.clone(), 0).await; let slot = state.slot(); let root = sync_committee_slot(&client, &state, &network, context, slot).await; // The next slot's block packs what was pooled over the head it builds on. + // It is asked for once that slot has started: the same head and pool on a + // clock one slot later. + let (client, _, network, _) = serve_fulu_pinned(Some(engine), sync_pool, 1).await; let next = slot + 1; let advanced = advance_to_slot(&state, next, &fulu_schedule()).unwrap(); let proposer = get_beacon_proposer_index(&advanced).unwrap(); @@ -1154,6 +1184,16 @@ mod gloas { async fn serve_gloas( clock_slot: u64, engine: Option, + ) -> Served { + serve_gloas_sharing(clock_slot, engine, SharedSyncCommitteePool::default()).await + } + + /// [`serve_gloas`] pooling sync committee messages in `sync_pool`; see + /// [`spawn_server_sharing`]. + async fn serve_gloas_sharing( + clock_slot: u64, + engine: Option, + sync_pool: SharedSyncCommitteePool, ) -> Served { let state = head_state(); let block = gloas_beacon_block( @@ -1170,7 +1210,8 @@ mod gloas { let head_root = header.hash_tree_root(); let store = beacon_store_with_head_block(state.clone(), config(), block, head_root, clock_slot); - let (client, network, payload_pool, _address) = spawn_server(store.clone(), engine).await; + let (client, network, payload_pool, _address) = + spawn_server_sharing(store.clone(), engine, sync_pool).await; let genesis = client.genesis().await.expect("genesis"); let spec = client .spec() @@ -1589,18 +1630,28 @@ mod gloas { #[tokio::test] async fn the_validator_client_can_serve_on_the_sync_committee_at_a_gloas_slot() { let engine = fake_execution_client(0).await; + let sync_pool = SharedSyncCommitteePool::default(); let Served { client, state, network, - store, context, .. - } = serve_gloas(32, Some(engine)).await; + } = serve_gloas_sharing(32, Some(engine.clone()), sync_pool.clone()).await; let client = Arc::new(client); let slot = state.slot(); let root = sync_committee_slot(&client, &state, &network, context.clone(), slot).await; + // The next slot's block, asked for once that slot has started: the + // same head and pool on a clock one slot later. + let Served { + client, + network, + store, + context, + .. + } = serve_gloas_sharing(33, Some(engine), sync_pool).await; + let client = Arc::new(client); let next = slot + 1; let advanced = advance_to_slot(&state, next, &config()).unwrap(); let proposer = get_beacon_proposer_index(&advanced).unwrap(); diff --git a/docs/metrics.md b/docs/metrics.md index 38680b44d..5f0b2227a 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -474,9 +474,9 @@ ethlambda-specific, not part of the leanMetrics spec. | Name | Type | Usage | Sample collection event | Labels | Buckets | |------|------|-------|-------------------------|--------|---------| | `lean_beacon_publish_data_columns_seconds` | Histogram | Time to compute and verify a published block's cells and build its data column sidecars | On every published block that carries blobs and whose cells verify | | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5 | -| `lean_beacon_builder_request_seconds` | Histogram | Time a builder API call took | On every `getHeader`, `registerValidator`, `submitBlindedBlockV2` and `status` call to the configured builder | call=get_header,register_validators,submit_blinded_block,status
outcome=ok,no_bid,timeout,unreachable,bad_request,unexpected_status,invalid_response,unsupported_fork,encode | 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 2, 5, 10 | +| `lean_beacon_builder_request_seconds` | Histogram | Time a builder API call took | On every `getHeader`, `registerValidator`, `submitBlindedBlockV2` and `status` call to the configured builder | call=get_header,register_validators,submit_blinded_block,status
outcome=ok,no_bid,timeout,unreachable,bad_request,unexpected_status,unexpected_success,invalid_response,unsupported_fork,encode | 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 2, 5, 10 | | `lean_beacon_builder_registrations_total` | Counter | Validator registrations received, by what became of them | On every `register_validator` request | outcome=forwarded,failed,dropped | | -| `lean_beacon_builder_bid_rejections_total` | Counter | Builder bids refused before selection, by reason | On every bid that fails validation or the state transition | reason=zero_value,parent_hash,prev_randao,timestamp,block_number,withdrawals,too_many_blobs,blob_gas,gas_limit,signature,state_transition | | +| `lean_beacon_builder_bid_rejections_total` | Counter | Builder bids refused before selection, by reason | On every bid that fails validation or the state transition | reason=parent_hash,prev_randao,timestamp,block_number,withdrawals,too_many_blobs,blob_gas,gas_limit,signature,state_transition | | | `lean_beacon_builder_payload_source_total` | Counter | Blocks produced with a builder configured, by where the payload came from | On every produced block when `--builder` is set | source=local,builder | | | `lean_beacon_builder_fallback_total` | Counter | Proposals the circuit breaker kept from the builder, by reason | On every proposal by a registered proposer that the breaker trips for | reason=syncing,optimistic_head,no_finality,consecutive_skips,skips_per_epoch | | diff --git a/docs/rpc.md b/docs/rpc.md index 4772c3027..f9aba76a5 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -443,7 +443,13 @@ the chain actor writes, so no request waits on the actor. malformed (its commitments, proofs and blobs disagree in count). A slot the schedule does not place at fulu (gloas included) is a `400`, checked before the execution client; `produceBlockV3` is the fulu endpoint, and a gloas slot - goes to `produceBlockV4`. + goes to `produceBlockV4`. So is a slot that has not started yet: one starting + more than `MAXIMUM_GOSSIP_CLOCK_DISPARITY` (500 ms) after this node's clock, + gossip's `is_future_slot`, is refused before any state is advanced to it. + Every block endpoint here applies the same bound, since advancing to a + far-future slot would run every epoch transition up to it. A validator client + asks once its slot has started, and a block further ahead would be ignored on + gossip anyway. **The builder path** is taken only when `--builder` is set, the proposer registered through this node (`register_validator`, remembered for two @@ -452,8 +458,8 @@ the chain actor writes, so no request waits on the actor. is too far behind, or too many slots were skipped (the `--builder-fallback-*` flags set the thresholds). The builder's `getHeader` is then asked concurrently with the local build, bounded by its own timeout, and an error - or a timeout means no bid. A bid is checked before it competes: a non-zero - value, the head's execution block as its parent, the proposal's `prev_randao` + or a timeout means no bid. A bid is checked before it competes: the head's + execution block as its parent, the proposal's `prev_randao` and timestamp, the parent's block number plus one, the withdrawals root, the blob commitment and blob gas bounds, a gas limit within one step of the parent's toward the registered target, and the builder's signature. Its fee recipient is not checked (see @@ -464,7 +470,9 @@ the chain actor writes, so no request waits on the actor. payload wins iff its value is at least `factor * bid / 100`, so a tie and a factor of 0 keep it and `u64::MAX` takes the bid; the execution client's `shouldOverrideBuilder` keeps the local payload whatever the bid pays. If one - side fails the other is served. A blinded answer is the blinded block as SSZ + side fails the other is served. A bid that pays nothing is not refused, but + it never beats a local payload, whatever the factor: it is served only when + the local payload or its block fails. A blinded answer is the blinded block as SSZ or JSON, with `Eth-Execution-Payload-Blinded: true` and `Eth-Execution-Payload-Value` set to the bid's value in wei. - **Graffiti** is the validator client's, with both clients' codes and @@ -483,8 +491,9 @@ the chain actor writes, so no request waits on the actor. content type is a `415`). The JSON is the specification's: lodestar's and nimbus's validator clients post it, teku's falls back to it, and lighthouse and prysm post SSZ. Both reach the same checks after decoding. Fulu: a - `SignedBlockContents`. A slot the schedule does not place at fulu is a `400`, - checked before any state is advanced. The block is then checked against its + `SignedBlockContents`. A slot the schedule does not place at fulu, or one that + has not started yet (see `produceBlockV3`), is a `400`, checked before any + state is advanced. The block is then checked against its **parent's** state advanced to its slot: an unknown parent, a slot not after the parent's, the wrong proposer or a bad proposer signature is a `400`. Then every blob's cells are computed and all cell proofs batch-verified on a @@ -495,7 +504,7 @@ the chain actor writes, so no request waits on the actor. last the chain actor gets this node's custody columns, then the block, to import. Gloas: a bare `gloas::SignedBeaconBlock` (a gloas block carries no payload or blobs, so no sidecars go with it; they follow in the envelope). The - slot must be scheduled at gloas, the parent must be held, the slot must be + slot must be scheduled at gloas and have started, the parent must be held, the slot must be after the parent's, and the proposer and signature are checked against the parent's state advanced to the slot (all `400` otherwise), before the block is gossiped and handed to the chain actor. This node publishes one block per @@ -506,7 +515,7 @@ the chain actor writes, so no request waits on the actor. - **`POST beacon/blinded_blocks`** takes a fulu `SignedBlindedBeaconBlock` as JSON or SSZ, by `Content-Type` as above, with `Eth-Consensus-Version: fulu`. It is checked as a fulu `beacon/blocks` is, before anything leaves this node: - the slot is scheduled at fulu, the parent is held and before the block, the + the slot is scheduled at fulu and has started, the parent is held and before the block, the proposer is the one the parent's state names for the slot, and the proposer's signature verifies (a blinded block has its full block's root, so the check is the same). Then the slot is claimed, shared with `beacon/blocks`, and the @@ -517,8 +526,13 @@ the chain actor writes, so no request waits on the actor. the one `beacon/blocks` has (`413` over it). The builder's answer is logged from the submission task, an error when it did not take a signed block. The status is the builder's: 202 becomes `200`, a builder `400` stays `400`, and anything else - is a `500`. There is no local fallback, since a second block would be an - equivocation. With no `--builder` the answer is `500`. + is a `500`. Only 202 is success, as builder-specs defines `submitBlindedBlockV2`; + a builder answering another 2xx (200 is what V1 answered) gets a `500` too, + with an error log of its own saying the builder may have published the block + anyway, so an operator can tell it from a refusal. There is no local fallback, since a second block would be an + equivocation. With no `--builder` the answer is `500`. `broadcast_validation` + is not read: the checks above run whatever it names, and the builder, not + this node, broadcasts the block. - **The operation pool** holds, beside the attestation pool and in memory in the same `Store` (`Store::operation_pool`): proposer slashings (one per proposer), attester slashings (one per `hash_tree_root`), voluntary exits and @@ -556,7 +570,7 @@ entries are decoded and not consulted. | `200` SSZ | Self-built with `include_payload=true`: `BlockContents` (`block`, `execution_payload_envelope`, `kzg_proofs`, `blobs`). Otherwise, and for every bid-won block, the bare `gloas::BeaconBlock` | | `200` JSON | `{version: "gloas", consensus_block_value, execution_payload_value, execution_payload_included, data}`, where `data` is the same container as the SSZ body | | Response headers | `Eth-Consensus-Version: gloas`, `Eth-Execution-Payload-Included` (`true` or `false`; always `false` for a bid-won block), `Eth-Execution-Payload-Value` (wei, decimal; a bid's value times `10^9` when it won), `Eth-Consensus-Block-Value` (always `0`: the comparison is of execution payload values only). No `Eth-Builder-Url`: that is for builder-API bids | -| `400` | A slot the schedule does not place at gloas, a missing or malformed query, a wrong `Eth-Consensus-Version`, a bad body, a slot not after the head block, or a `randao_reveal` that does not verify against the slot's proposer. The slot check precedes the execution-client check, so it is a `400` on any node | +| `400` | A slot the schedule does not place at gloas or that has not started yet (more than 500 ms ahead of this node's clock, as for `produceBlockV3`), a missing or malformed query, a wrong `Eth-Consensus-Version`, a bad body, a slot not after the head block, or a `randao_reveal` that does not verify against the slot's proposer. The slot check precedes the execution-client check, so it is a `400` on any node | | `503` | No execution client and no viable pooled bid for the slot, the execution client did not start or return a build (and no bid took its place), the node is building on a FULL parent whose envelope it does not hold, or the winning bid failed to build and there is no local payload to fall back on | The node advances the head state to the slot, and decides which parent payload From 086f50c43dbcb244bb26fcbf02c074df48b83aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:52:01 -0300 Subject: [PATCH 15/20] refactor(rpc): count the breaker's consecutive skips from the head block, and document the unchecked builder payment - The consecutive-skips check walked back through `block_roots` one slot at a time until it found a block, with no cap. The run is now the distance from the head block, which the advanced proposal state keeps as its latest block header: `slot - 1 - latest_block_header.slot`, Lighthouse's `head_skips`. The per-epoch count still reads `block_roots`, over a fixed 32 slots. A new test advances a real state through slot processing to pin that the header stays at the head block. - `bid::validate` now says, where the check would go, why a bid's fee recipient is not compared with the registered one (builders name themselves and pay the proposer with the payload's last transaction), that the payment cannot be checked there either (the header carries no transactions, and the builder publishes the block after signing), and that the relay checks it when it validates a builder's submission. spec_deviations.md says the same at length; rpc.md and the debug log's comment point at it. Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- crates/net/rpc/src/beacon/builder_flow/bid.rs | 18 +++++-- .../rpc/src/beacon/builder_flow/breaker.rs | 47 +++++++++++++++++-- crates/net/rpc/src/beacon/proposal.rs | 5 +- docs/rpc.md | 4 +- docs/spec_deviations.md | 31 ++++++++++-- 5 files changed, 90 insertions(+), 15 deletions(-) diff --git a/crates/net/rpc/src/beacon/builder_flow/bid.rs b/crates/net/rpc/src/beacon/builder_flow/bid.rs index 48f2b8366..4c16765e1 100644 --- a/crates/net/rpc/src/beacon/builder_flow/bid.rs +++ b/crates/net/rpc/src/beacon/builder_flow/bid.rs @@ -5,9 +5,10 @@ //! withdrawals, blob count) are checked here too, before //! `assemble_blinded_block` repeats them as a backstop, so a refusal has a //! reason to count and costs no state transition. What is left unchecked -//! (the payload's execution validity, its block hash and the fields only an -//! execution client could verify) is trusted to the relay, as every client -//! does: the payload is revealed only after the proposer signs. +//! (the payload's execution validity, its block hash, the fields only an +//! execution client could verify, and the fee recipient and the payment to the +//! proposer, see [`validate`]) is trusted to the relay, as every client does: +//! the payload is revealed only after the proposer signs. use ethlambda_state_transition::beacon::block_production::PayloadInputs; use ethlambda_state_transition::beacon::bls; @@ -113,6 +114,17 @@ pub(crate) fn validate(bid: &SignedBuilderBid, context: &BidContext) -> Result<( ) { return Err(Rejection::GasLimit); } + // Not checked: `header.fee_recipient` against the registered one, which + // builder-specs' `process_bid` asserts. A builder names itself as the + // payload's fee recipient, so fees and MEV accrue to it, and pays the + // proposer with the payload's last transaction, to the registered address + // and worth the bid's value. A mismatch is therefore the normal case, and + // the assertion would refuse nearly every real bid; no client reviewed + // makes it. Nor can the payment be checked here: the header carries no + // transactions, and the builder publishes the block after the proposer + // signs. The relay checks it, when it validates the builder's submission + // on an execution node before serving the bid. See + // `docs/spec_deviations.md`. let domain = compute_builder_domain(context.genesis_fork_version); let root = compute_signing_root(message.hash_tree_root(), domain); if !bls::verify(&message.pubkey, root, &bid.signature) { diff --git a/crates/net/rpc/src/beacon/builder_flow/breaker.rs b/crates/net/rpc/src/beacon/builder_flow/breaker.rs index 07f5129b9..2f6cb33ff 100644 --- a/crates/net/rpc/src/beacon/builder_flow/breaker.rs +++ b/crates/net/rpc/src/beacon/builder_flow/breaker.rs @@ -3,9 +3,11 @@ //! //! builder-specs leaves the conditions to the client; these follow //! Lighthouse's. They read the proposal state alone, so two nodes holding -//! the same state agree: a slot `i` before the proposal slot was empty iff -//! `block_roots[i] == block_roots[i - 1]`, since `process_slots` carries the -//! previous root forward over an empty slot. +//! the same state agree. The empty slots right before the proposal are the +//! ones since the head block, which the state keeps as its latest block +//! header. Across the epoch's worth of slots before the proposal, a slot `i` +//! was empty iff `block_roots[i] == block_roots[i - 1]`, since +//! `process_slots` carries the previous root forward over an empty slot. use ethlambda_state_transition::beacon::helpers::accessors::get_block_root_at_slot; use ethlambda_types::beacon::containers::BeaconState; @@ -87,7 +89,10 @@ pub(crate) fn check( if since_finalization > thresholds.epochs_since_finalization { return Err(Trip::NoFinality); } - let consecutive = (1..slot).rev().take_while(|&i| is_empty(state, i)).count() as u64; + // Every slot between the head block and the proposal was empty, so the + // run is the distance between them: Lighthouse's `head_skips`. + let head_slot = state.latest_block_header().slot; + let consecutive = slot.saturating_sub(1).saturating_sub(head_slot); if consecutive > thresholds.skips { return Err(Trip::ConsecutiveSkips); } @@ -121,7 +126,8 @@ mod tests { use super::*; /// A fulu state at `slot` whose every earlier slot had a block except - /// those in `empty`, finalized at `finalized_epoch`. + /// those in `empty`, finalized at `finalized_epoch`. Its latest block + /// header is the last of those blocks, as slot processing leaves it. fn state_at(slot: Slot, empty: &[Slot], finalized_epoch: Epoch) -> BeaconState { let mut state = with_signing_validators_at(ForkName::Fulu, 8); let BeaconState::Fulu(fulu) = &mut state else { @@ -136,6 +142,7 @@ mod tests { let mut bytes = [0u8; 32]; bytes[..8].copy_from_slice(&(i + 1).to_le_bytes()); root = H256(bytes); + fulu.latest_block_header.slot = i; } fulu.block_roots[(i % history) as usize] = root; } @@ -217,6 +224,36 @@ mod tests { assert!(is_empty(&state, slot - 1)); } + /// Slot processing leaves the latest block header at the head block, so + /// the run counted from it holds on a state advanced the real way. + #[test] + fn the_run_is_counted_from_the_head_block_slot_processing_leaves() { + use ethlambda_state_transition::beacon::block_production::advance_to_slot; + use ethlambda_types::beacon::config::Config; + + let config = Config::mainnet().with_fork_epoch(ForkName::Fulu, 0); + // A head state as its block leaves it: the header at the state's slot. + let mut head = with_signing_validators_at(ForkName::Fulu, 8); + let BeaconState::Fulu(fulu) = &mut head else { + unreachable!("built as fulu") + }; + fulu.latest_block_header.slot = fulu.slot; + let head_slot = head.slot(); + // The fixture's history is all zero roots, so only the consecutive + // count is judged here. + let thresholds = FallbackThresholds { + skips_per_epoch: u64::MAX, + ..FallbackThresholds::DEFAULT + }; + let check_at = |slot| { + let state = advance_to_slot(&head, slot, &config).unwrap(); + check(&state, &thresholds, false, false) + }; + // Slots head+1 to head+3 empty: three in a row. Then four. + assert_eq!(check_at(head_slot + 4), Ok(())); + assert_eq!(check_at(head_slot + 5), Err(Trip::ConsecutiveSkips)); + } + #[test] fn an_empty_run_that_does_not_end_right_before_the_proposal_is_not_consecutive() { // Five empty slots, but slot 99 had a block. diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index 39816dfb0..567c4fe17 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -977,8 +977,9 @@ fn accepted_bid( } let message = bid.bid.message; if message.header.fee_recipient != request.registration.fee_recipient { - // Expected: relays pay the proposer with a transaction in the payload - // and name the builder as its fee recipient (docs/spec_deviations.md). + // Expected, so not a refusal: the builder names itself and pays the + // registered address with a transaction. Why `bid::validate` does not + // check either is written where the check would go. debug!(%slot, "The bid's fee recipient is not the registered one"); } Some(message) diff --git a/docs/rpc.md b/docs/rpc.md index f9aba76a5..4a39f8391 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -462,7 +462,9 @@ the chain actor writes, so no request waits on the actor. execution block as its parent, the proposal's `prev_randao` and timestamp, the parent's block number plus one, the withdrawals root, the blob commitment and blob gas bounds, a gas limit within one step of the parent's toward the registered target, - and the builder's signature. Its fee recipient is not checked (see + and the builder's signature. Its fee recipient is not checked, since builders + name themselves there and pay the proposer with a transaction this node never + sees before signing; the relay checks that payment (see `spec_deviations.md`). A bid that passes goes through `assemble_blinded_block` with the packed attestations, operations and sync aggregate, so the blinded block's state root comes from running the block through `process_block`. diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 8e04fb262..1da097fb4 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -133,10 +133,33 @@ from the Beacon API, and no other way. - **The specification:** builder-specs' `process_bid` asserts `header.fee_recipient == fee_recipient`. - **ethlambda:** a mismatch is logged at `debug` only. -- **Why:** relays pay the proposer with a transaction in the payload and name - the builder's address as the coinbase, so the assertion would refuse nearly - every real bid. Every client reviewed (Lighthouse, Teku, Prysm, Nimbus, - Lodestar, Grandine) skips it. +- **Why:** a builder names its own address as the payload's fee recipient + (the coinbase), so priority fees and MEV accrue to it, and pays the proposer + with a transaction to the registered fee recipient, worth the bid's `value`, + as the payload's last transaction. A mismatch is therefore the normal case, + and the assertion would refuse nearly every real bid. Every client reviewed + (Lighthouse, Teku, Prysm, Nimbus, Lodestar, Grandine) skips it. A counter of + mismatches would rise on nearly every bid and tell an operator nothing. +- **The payment itself is not checked either, by this node or any client + reviewed.** A bid carries only the payload's header: the transactions, + payment included, stay with the builder until the proposer has signed. After + signing, `submitBlindedBlockV2` returns no payload; the builder publishes + the block, which reaches this node over gossip like any other, by which time + nothing can be undone. +- **Who does check it:** the relay. Before serving a builder's bid, a relay + validates the submission on an execution node, and that validation requires + the proposer payment. Flashbots' block-validation API + (`flashbots_validateBuilderSubmission*`) passes the registered fee + recipient and the bid's value to `ValidatePayload`, which requires the + block's last transaction to pay that address at least that much (or, if the + relay configures it, the address's balance to rise by it). The proposer + relies on the relay for this, as for the payload's validity. A relay + publishes what it delivered on its data API + (`/relay/v1/data/bidtraces/proposer_payload_delivered`), which is where an + underpayment would surface. An operator who wants more than that trust must + choose relays accordingly. A check after the fact, comparing a published + block's last transaction with the bid it won, is possible but not + implemented. ## Fee recipient and gas target come from proposer preferences, with fallbacks From d0b67525729e3597a7b9d38562c31cd75fb57428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:20:49 -0300 Subject: [PATCH 16/20] fix(rpc): scale the boost factor before rounding, and remember registrations only once the builder accepts them - `choose` compared the local payload with `factor * (bid / 100)`, the beacon-APIs text's order, which rounds the bid down to a multiple of 100 wei before weighing it: at the default factor a bid up to 99 wei better than the local payload lost. It now uses `factor * bid / 100`, rounded down once at the end and saturating at `u128::MAX` (`choice::boosted`, split so no type wider than `u128` is needed). This matches Teku, Lodestar, Nimbus and our own gloas `choose_payload`; spec_deviations.md records the departure from the spec's order. - `register_validator` recorded the kept registrations for block production before forwarding them, whatever the builder answered, and this node does not verify their signatures. A registration the builder refused (a forged `gas_limit = 1` under a later timestamp, say) replaced the accepted one and made every real bid fail the gas-limit check. They are now recorded only once the builder accepts them, inside the forwarding task, as Prysm, Teku and Lodestar do. Fee recipients are still set either way. Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- .../net/rpc/src/beacon/builder_flow/cache.rs | 5 +- .../net/rpc/src/beacon/builder_flow/choice.rs | 88 ++++++++++++++++--- crates/net/rpc/src/beacon/registrations.rs | 75 +++++++++++++--- docs/rpc.md | 10 ++- docs/spec_deviations.md | 21 +++++ 5 files changed, 167 insertions(+), 32 deletions(-) diff --git a/crates/net/rpc/src/beacon/builder_flow/cache.rs b/crates/net/rpc/src/beacon/builder_flow/cache.rs index 8ccbb7ecb..82eb3162c 100644 --- a/crates/net/rpc/src/beacon/builder_flow/cache.rs +++ b/crates/net/rpc/src/beacon/builder_flow/cache.rs @@ -1,6 +1,5 @@ -//! The registrations this node forwarded to its builder, so block -//! production knows which proposers use the builder, and with what gas -//! limit. +//! The registrations this node's builder accepted, so block production knows +//! which proposers use the builder, and with what gas limit. use std::collections::HashMap; use std::sync::{Arc, Mutex}; diff --git a/crates/net/rpc/src/beacon/builder_flow/choice.rs b/crates/net/rpc/src/beacon/builder_flow/choice.rs index 2c403faae..22f31e6c7 100644 --- a/crates/net/rpc/src/beacon/builder_flow/choice.rs +++ b/crates/net/rpc/src/beacon/builder_flow/choice.rs @@ -31,12 +31,12 @@ pub(crate) struct LocalOffer { /// Which payload the block carries, or `None` when neither is usable. /// -/// beacon-APIs' `builder_boost_factor`, taken literally: the local payload -/// wins iff `local >= factor * (bid / 100)`. So 100 compares the two as they -/// are, 0 always keeps the local payload, and a tie goes local. -/// A factor of `u64::MAX` always takes the builder, except that the execution -/// client's `shouldOverrideBuilder` still wins. When one side failed, the -/// other stands. +/// beacon-APIs' `builder_boost_factor`, a percentage: the local payload wins +/// iff `local >= factor * bid / 100`, multiplied first and rounded down once +/// (see [`boosted`]). So 100 compares the two exactly as they are, 0 always +/// keeps the local payload, and a tie goes local. A factor of `u64::MAX` +/// always takes the builder, except that the execution client's +/// `shouldOverrideBuilder` still wins. When one side failed, the other stands. /// /// A bid that pays nothing never beats a local payload, whatever the factor: /// it stands only in for one that failed, here or when its block fails to @@ -53,17 +53,32 @@ pub(crate) fn choose( (Some(_), Some(0)) => Some(Source::Local), (Some(local), Some(_)) if local.should_override_builder => Some(Source::Local), (Some(_), Some(_)) if boost_factor == u64::MAX => Some(Source::Builder), - (Some(local), Some(bid)) => { - let boosted = u128::from(boost_factor).saturating_mul(bid / 100); - Some(if local.value >= boosted { - Source::Local - } else { - Source::Builder - }) - } + (Some(local), Some(bid)) => Some(if local.value >= boosted(bid, boost_factor) { + Source::Local + } else { + Source::Builder + }), } } +/// `factor * bid / 100`, rounded down once, at the end, and saturating at +/// `u128::MAX` as beacon-APIs asks of a large factor. +/// +/// The spec's text divides first, `factor * (bid / 100)`, which rounds the bid +/// down to a multiple of 100 wei before scaling it: even at the default factor +/// of 100, a bid up to 99 wei better than the local payload lost to it. +/// Multiplying first is exact. The product is split as +/// `factor * (bid / 100) + factor * (bid % 100) / 100`, which is the same +/// number since the first term is a whole multiple of 100, so no type wider +/// than `u128` is needed: the second term is below `factor`, and only the +/// first can overflow. +fn boosted(bid: u128, factor: u64) -> u128 { + let factor = u128::from(factor); + let whole = factor.saturating_mul(bid / 100); + let rest = factor * (bid % 100) / 100; + whole.saturating_add(rest) +} + #[cfg(test)] mod tests { use super::*; @@ -108,6 +123,21 @@ mod tests { 100, Some(Source::Local), ), + ( + "one wei more wins at 100", + local(ETH), + Some(ETH + 1), + 100, + Some(Source::Builder), + ), + ( + "rounded once, after scaling", + // 50 * 3 / 100 is 1; 50 * (3 / 100) would be 0. + local(0), + Some(3), + 50, + Some(Source::Builder), + ), ( "factor 0 keeps local", local(1), @@ -169,4 +199,34 @@ mod tests { assert_eq!(choose(local, bid, factor), expected, "{name}"); } } + + /// `factor * bid / 100` rounded down once, checked against the product + /// itself wherever it fits in a `u128`. + #[test] + fn the_bid_is_scaled_before_it_is_rounded() { + let bids = [ + 0, + 1, + 99, + 100, + 101, + 12_345, + ETH - 1, + ETH + 99, + u128::from(u64::MAX), + ]; + let factors = [0, 1, 50, 99, 100, 101, 150, 200, 1_000_000, u64::MAX - 1]; + for bid in bids { + for factor in factors { + let exact = u128::from(factor) * bid / 100; + assert_eq!(boosted(bid, factor), exact, "{factor} * {bid} / 100"); + } + } + } + + #[test] + fn a_product_past_u128_saturates() { + assert_eq!(boosted(u128::MAX, 200), u128::MAX); + assert_eq!(boosted(u128::MAX / 2, u64::MAX - 1), u128::MAX); + } } diff --git a/crates/net/rpc/src/beacon/registrations.rs b/crates/net/rpc/src/beacon/registrations.rs index f90124571..5fa9df2c6 100644 --- a/crates/net/rpc/src/beacon/registrations.rs +++ b/crates/net/rpc/src/beacon/registrations.rs @@ -9,9 +9,10 @@ //! not reach the builder, not that the request fails. Each kept //! registration also becomes its validator's fee recipient, as //! `prepare_beacon_proposer` would make it, so a block built locally after -//! all still pays the address the validator registered. The kept -//! registrations are also remembered for block production, which asks the -//! builder only for proposers that registered. +//! all still pays the address the validator registered. Once the builder +//! accepts them, the kept registrations are also remembered for block +//! production, which asks the builder only for proposers that registered. Not +//! before: this node does not verify their signatures, the builder does. use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -98,23 +99,29 @@ async fn register( recipients.insert(*index, registration.message.fee_recipient); } } - // Remembered at the epoch the filter judged them at, so block production - // sees them for as long as the validator client keeps sending them. - builder.registrations.record( - kept.iter().map(|(_, registration)| ®istration.message), - epoch, - ); let kept: Vec = kept .into_iter() .map(|(_, registration)| registration) .collect(); // On its own task: a validator client that hangs up drops this handler, - // and the forward, its count and its log must complete regardless. + // and the forward, its count, its log and the record must complete + // regardless. let forwarded = tokio::spawn(async move { let count = kept.len() as u64; let result = builder.client.register_validators(&kept).await; match &result { - Ok(()) => inc_builder_registrations("forwarded", count), + Ok(()) => { + inc_builder_registrations("forwarded", count); + // Remembered only once the builder took them: it verifies + // their signatures and this node does not, so one it refused + // (a forged gas limit, say) must not steer this node's bids. + // At the epoch the filter judged them at, so block production + // sees them for as long as the validator client keeps sending + // them. + builder + .registrations + .record(kept.iter().map(|registration| ®istration.message), epoch); + } Err(err) => { inc_builder_registrations("failed", count); warn!(%err, count, "Builder did not take the registrations"); @@ -466,9 +473,9 @@ mod tests { let (status, _) = post( store, - Some(builder), + Some(builder.clone()), fee_recipients.clone(), - json_request(&[registration]), + json_request(std::slice::from_ref(®istration)), ) .await; @@ -477,6 +484,48 @@ mod tests { fee_recipients.lock().unwrap().get(&0), Some(&ExecutionAddress::repeat_byte(0x11)) ); + // The builder never took it, so block production never asks for it. + assert_eq!( + builder.registrations.get(®istration.message.pubkey, 0), + None + ); + } + + /// The review's attack: a registration under a later timestamp with a + /// forged gas limit, which the builder refuses since its signature does + /// not verify. It must not replace the one the builder accepted, or every + /// real bid would fail the gas-limit check. + #[tokio::test] + async fn a_registration_the_builder_refuses_is_not_remembered() { + let store = store_with_exited(&[]); + let relay = MockRelay::start(genesis_fork_version(&store)).await; + let builder = client_for(&relay).unwrap(); + let genuine = signed_registration(0, genesis_fork_version(&store)); + let (status, body) = post( + store.clone(), + Some(builder.clone()), + FeeRecipients::default(), + json_request(std::slice::from_ref(&genuine)), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + + let mut forged = genuine.clone(); + forged.message.gas_limit = 1; + forged.message.timestamp += 1; + let (status, _) = post( + store, + Some(builder.clone()), + FeeRecipients::default(), + json_request(&[forged]), + ) + .await; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + builder.registrations.get(&genuine.message.pubkey, 0), + Some(genuine.message) + ); } #[tokio::test] diff --git a/docs/rpc.md b/docs/rpc.md index 4a39f8391..d3902da35 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -421,7 +421,11 @@ the chain actor writes, so no request waits on the actor. registration also sets its validator's fee recipient, so a block built locally still pays the registered address. The kept ones go to `--builder` in one call, skipped when none are left. A builder's 400 is passed through and - any other failure is a 500; with no `--builder` the answer is 500. + any other failure is a 500; with no `--builder` the answer is 500. Only + once the builder accepts them are they remembered for block production + (the proposer asks the builder for a bid, against the registered gas + limit): the builder verifies their signatures and this node does not, so a + registration it refuses must not steer this node's bids. - **`blocks/{slot}`** advances the head state to the slot and asks the node's own execution client to build on the head (`forkchoiceUpdatedV3` with payload attributes, then `getPayloadV5`), with the proposer's fee recipient. @@ -469,7 +473,9 @@ the chain actor writes, so no request waits on the actor. with the packed attestations, operations and sync aggregate, so the blinded block's state root comes from running the block through `process_block`. `builder_boost_factor` (percent, default 100) then weighs the two: the local - payload wins iff its value is at least `factor * bid / 100`, so a tie and a + payload wins iff its value is at least `factor * bid / 100`, multiplied + first and rounded down once (the spec's text rounds the bid first; see + `spec_deviations.md`), so a tie and a factor of 0 keep it and `u64::MAX` takes the bid; the execution client's `shouldOverrideBuilder` keeps the local payload whatever the bid pays. If one side fails the other is served. A bid that pays nothing is not refused, but diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 1da097fb4..aeb38b321 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -161,6 +161,27 @@ from the Beacon API, and no other way. block's last transaction with the bid it won, is possible but not implemented. +## `builder_boost_factor` scales the bid before rounding + +- **The specification:** beacon-APIs' `produceBlockV3` serves the local payload + iff `exec_node_payload_value >= builder_boost_factor * + (builder_payload_value // 100)`: the bid is rounded down to a multiple of + 100 wei, then scaled. +- **ethlambda:** the local payload wins iff its value is at least + `(builder_boost_factor * builder_payload_value) // 100`, scaled first and + rounded down once (`builder_flow::choice::boosted`), saturating at + `u128::MAX`. A tie still goes local, and the factors 0, 100 and `2^64 - 1` + keep the meanings the specification gives them. +- **Why:** rounding first loses up to 99 wei of the bid before it is weighed, + so even at the default factor of 100 a bid slightly better than the local + payload lost to it. Scaling first makes 100 an exact comparison. The two + differ by less than `factor` wei, only ever in the bid's favour. The gloas + `produceBlockV4` comparison (`bid_selection::choose_payload`) already + multiplies first. Teku and Lodestar compare `local * 100` against + `factor * bid`, and Nimbus computes `(factor * bid) div 100` (giving a tie to + the builder). Lighthouse follows the specification's order but skips the + arithmetic at factor 100 for the same reason (sigp/lighthouse#5352). + ## Fee recipient and gas target come from proposer preferences, with fallbacks - **The specification:** the payload is built toward the `target_gas_limit` of From dc7e26f23935164717776bc2e5e04ac78ce8812b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:24:33 -0300 Subject: [PATCH 17/20] feat(rpc): warn when the builder's block is served under shouldOverrideBuilder The override always puts the local block first, so the builder's block is reached under it only once the local one failed to build. That fallback stays, since a block beats a missed slot, but it now logs a warning saying the execution client's override was set aside. `local_block` also logs why assembly failed, which it dropped before, so the cause shows next to it. Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- crates/net/rpc/src/beacon/proposal.rs | 22 ++++++++++++++++++++-- docs/rpc.md | 4 +++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index 567c4fe17..64a1f87b8 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -674,6 +674,7 @@ async fn produce( should_override_builder: parts.built.should_override_builder, }); let bid_value = bid.as_ref().map(|bid| wei_u128(&bid.value)); + let overriding = local_offer.is_some_and(|offer| offer.should_override_builder); let order = if choice::choose(local_offer, bid_value, boost_factor) == Some(Source::Builder) { [Source::Builder, Source::Local] } else { @@ -700,6 +701,17 @@ async fn produce( options, ); if let Some(blinded) = blinded { + // The override always puts the local block first, so the + // builder is reached under it only once that block failed. + // A block still beats a missed slot, but the execution + // client asked for none of the builder's. + if overriding { + warn!( + %slot, + proposer, + "The execution client set shouldOverrideBuilder, but the local block failed to build; serving the builder's block instead" + ); + } inc_payload_source(Source::Builder.label()); let value = decimal(&blinded.value); info!(%slot, proposer, %value, "Produced a blinded block around the builder's bid"); @@ -886,9 +898,15 @@ fn local_block( config, options, ) - .map_err(|_| ApiError::Internal("the block failed to build"))? + .map_err(|err| { + warn!(%slot, %err, "The local block failed to build"); + ApiError::Internal("the block failed to build") + })? + } + Err(err) => { + warn!(%slot, %err, "The local block failed to build"); + return Err(ApiError::Internal("the block failed to build")); } - Err(_) => return Err(ApiError::Internal("the block failed to build")), }; let contents = FuluBlockContents { block, diff --git a/docs/rpc.md b/docs/rpc.md index d3902da35..7eae4fd6a 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -478,7 +478,9 @@ the chain actor writes, so no request waits on the actor. `spec_deviations.md`), so a tie and a factor of 0 keep it and `u64::MAX` takes the bid; the execution client's `shouldOverrideBuilder` keeps the local payload whatever the bid pays. If one - side fails the other is served. A bid that pays nothing is not refused, but + side fails the other is served, even under the override: a local block that + fails to build then gives way to the builder's, with a warning, since a + block beats a missed slot. A bid that pays nothing is not refused, but it never beats a local payload, whatever the factor: it is served only when the local payload or its block fails. A blinded answer is the blinded block as SSZ or JSON, with `Eth-Execution-Payload-Blinded: true` and From 60c697a6f24c1586b547a0ed88227669c0a39e9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:53:06 -0300 Subject: [PATCH 18/20] fix(rpc): bound the local build against a valid bid, refuse publishes behind finality, escape builder error text Bot review follow-ups on the fulu builder flow: - produceBlockV3: once a builder bid has validated, the local payload has LOCAL_BUILD_DEADLINE (2 s, Lodestar's race cutoff and Lighthouse's getPayload timeout) before it is given up on with a warning and the bid is served through the existing fallback. Without a valid bid the local build is still waited for, since giving up then only loses the block. - engine: getPayloadV5/V6 are one attempt at ENGINE_GET_PAYLOAD_TIMEOUT (1 s) instead of the 3 x 8 s ladder. execution-apis #958 dropped the method's timeout ("MUST respond immediately"); 1 s is the value it replaced and what Nimbus and Prysm (pre-gloas) use. - publishBlockV2 (fulu and gloas) and publishBlindedBlockV2 refuse, before advancing the parent's state, a block at or below the finalized slot and a parent at or below it that is not the finalized block itself (gossip's IGNORE and REJECT rules), so an unauthenticated request cannot make the node advance an old held state for epochs. - builder error text escapes every control character and U+2028/U+2029, not just CR/LF, and the 1 KiB cap never cuts an escape in two. - docs: the bid signature is checked against the key the bid names, which guards integrity, not the builder's identity; say so, with the other clients' behaviour (all verifying clients do the same). Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- crates/net/engine/src/client.rs | 50 ++- crates/net/engine/tests/wire_smoke.rs | 61 +++ crates/net/rpc/src/beacon/blinded_blocks.rs | 17 +- .../net/rpc/src/beacon/builder_client/mod.rs | 41 +- .../rpc/src/beacon/builder_client/tests.rs | 41 ++ crates/net/rpc/src/beacon/builder_flow/bid.rs | 24 ++ crates/net/rpc/src/beacon/proposal.rs | 388 ++++++++++++++++-- .../rpc/src/beacon/validator_client_tests.rs | 49 ++- crates/net/rpc/src/lib.rs | 14 + docs/beacon_engine.md | 8 + docs/rpc.md | 34 +- 11 files changed, 653 insertions(+), 74 deletions(-) diff --git a/crates/net/engine/src/client.rs b/crates/net/engine/src/client.rs index fb16ba6db..e34fef2de 100644 --- a/crates/net/engine/src/client.rs +++ b/crates/net/engine/src/client.rs @@ -26,7 +26,26 @@ pub const ENGINE_TIMEOUT: Duration = Duration::from_secs(8); /// for the older methods. pub const ENGINE_NEW_PAYLOAD_V5_TIMEOUT: Duration = Duration::from_secs(6); -/// How many times one call is attempted before it is given up on. +/// Timeout for `engine_getPayloadV5` and `engine_getPayloadV6`, which get this +/// one attempt and no retry. +/// +/// execution-apis #958 (merged 2026-10-26) dropped `getPayload`'s timeout: the +/// method "MUST respond immediately", and the consensus client "MAY abort such +/// a call at its own discretion". One second is the value it replaced, and the +/// one Nimbus uses (`GETPAYLOAD_TIMEOUT`, `el/el_manager.nim:53`) and Prysm +/// uses before gloas (300 ms from gloas, `execution/engine_jsonrpc.go:116-118` +/// and `:300-304`). Lighthouse (`engine_api/http.rs:52`) and Teku +/// (`AbstractExecutionEngineClient.java:85`) allow 2 s. +/// +/// Not retried, unlike every other method: the payload is due at the start of +/// the slot, so an answer a retry ladder could still get (seconds later) is +/// one the block cannot use, and an execution client may stop the build +/// process once it has served the call, so a retry after a lost answer can +/// find the payload gone. +pub const ENGINE_GET_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(1); + +/// How many times one call is attempted before it is given up on, for every +/// method but `getPayload` (see [`ENGINE_GET_PAYLOAD_TIMEOUT`]). /// /// Three, not the ten `ethlambda-p2p` uses for a peer fetch. Those are different /// numbers measuring different things: a peer ladder is bounded by a millisecond @@ -101,11 +120,23 @@ impl EngineClient { method: &str, params: serde_json::Value, timeout: Duration, + ) -> Result { + self.call_with_attempts(method, params, timeout, ENGINE_MAX_ATTEMPTS) + .await + } + + /// [`Self::call_with_timeout`], attempted at most `attempts` times. + async fn call_with_attempts( + &self, + method: &str, + params: serde_json::Value, + timeout: Duration, + attempts: u32, ) -> Result { let mut backoff = ENGINE_INITIAL_BACKOFF; let mut last = EngineError::Transport("no attempt was made".to_string()); - for attempt in 1..=ENGINE_MAX_ATTEMPTS { + for attempt in 1..=attempts { match self.call_once(method, ¶ms, timeout).await { Ok(value) => return Ok(value), // An answer, not a failure to get one. Do not retry. @@ -115,7 +146,7 @@ impl EngineClient { last = err; } } - if attempt < ENGINE_MAX_ATTEMPTS { + if attempt < attempts { tokio::time::sleep(backoff).await; backoff *= BACKOFF_MULTIPLIER; } @@ -301,12 +332,16 @@ impl EngineClient { /// `engine_getPayloadV5`: the payload a build process has assembled so /// far, with its blobs bundle and execution requests. + /// + /// One attempt at [`ENGINE_GET_PAYLOAD_TIMEOUT`], never retried; see there + /// for why. pub async fn get_payload( &self, payload_id: crate::building::PayloadId, ) -> Result { + let params = json!([payload_id]); let response: crate::building::GetPayloadV5Response = self - .call("engine_getPayloadV5", json!([payload_id])) + .call_with_attempts("engine_getPayloadV5", params, ENGINE_GET_PAYLOAD_TIMEOUT, 1) .await?; response.try_into() } @@ -331,12 +366,17 @@ impl EngineClient { /// `engine_getPayloadV6`: [`Self::get_payload`] for Amsterdam, answering /// with `ExecutionPayloadV4` (`blockAccessList` and `slotNumber` included). + /// + /// One attempt at [`ENGINE_GET_PAYLOAD_TIMEOUT`], like V5. Prysm tightens + /// its own to 300 ms from gloas; this keeps the second until an execution + /// client's gloas build times are known. pub async fn get_payload_v6( &self, payload_id: crate::building::PayloadId, ) -> Result { + let params = json!([payload_id]); let response: crate::building::GetPayloadV6Response = self - .call("engine_getPayloadV6", json!([payload_id])) + .call_with_attempts("engine_getPayloadV6", params, ENGINE_GET_PAYLOAD_TIMEOUT, 1) .await?; response.try_into() } diff --git a/crates/net/engine/tests/wire_smoke.rs b/crates/net/engine/tests/wire_smoke.rs index 72d943efd..affd897da 100644 --- a/crates/net/engine/tests/wire_smoke.rs +++ b/crates/net/engine/tests/wire_smoke.rs @@ -221,3 +221,64 @@ async fn forkchoice_updated_v4_sends_three_params_and_parses_syncing() { // The attributes are null and the bitmap follows them. assert!(request.contains(",null,\"0x01020000000000000000000000000000\"]")); } + +/// Accepts every connection, reads its request, and never answers. Returns +/// the endpoint and a count of the connections it took. +async fn serve_silently() -> (String, Arc) { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("an ephemeral loopback port"); + let addr = listener.local_addr().expect("the bound address"); + let connections = Arc::new(AtomicUsize::new(0)); + let counter = Arc::clone(&connections); + + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((socket, _)) = listener.accept().await { + counter.fetch_add(1, Ordering::SeqCst); + // Kept open, so the client waits on an answer rather than seeing + // the connection close. + held.push(socket); + } + }); + + (format!("http://{addr}"), connections) +} + +/// `getPayload` is one attempt at its own timeout: a ladder of three would +/// take seconds more and open a connection per attempt. +#[tokio::test] +async fn get_payload_gives_up_after_one_attempt_at_its_own_timeout() { + use ethlambda_engine::building::PayloadId; + use ethlambda_engine::client::ENGINE_GET_PAYLOAD_TIMEOUT; + use std::sync::atomic::Ordering; + use std::time::{Duration, Instant}; + + for v6 in [false, true] { + let (endpoint, connections) = serve_silently().await; + let client = client(endpoint); + let started = Instant::now(); + let err = if v6 { + client.get_payload_v6(PayloadId([1; 8])).await.map(drop) + } else { + client.get_payload(PayloadId([1; 8])).await.map(drop) + } + .expect_err("the mock never answers"); + let elapsed = started.elapsed(); + + assert!( + matches!(err, EngineError::Timeout(timeout) if timeout == ENGINE_GET_PAYLOAD_TIMEOUT), + "v6 {v6}: {err:?}" + ); + assert!( + elapsed >= ENGINE_GET_PAYLOAD_TIMEOUT, + "v6 {v6}: {elapsed:?}" + ); + // A second attempt would add the backoff and another full timeout. + let retry_floor = ENGINE_GET_PAYLOAD_TIMEOUT * 2 + Duration::from_millis(500); + assert!(elapsed < retry_floor, "v6 {v6}: {elapsed:?}"); + assert_eq!(connections.load(Ordering::SeqCst), 1, "v6 {v6}"); + } +} diff --git a/crates/net/rpc/src/beacon/blinded_blocks.rs b/crates/net/rpc/src/beacon/blinded_blocks.rs index bc7800a7d..df9ea6297 100644 --- a/crates/net/rpc/src/beacon/blinded_blocks.rs +++ b/crates/net/rpc/src/beacon/blinded_blocks.rs @@ -27,7 +27,9 @@ use tracing::{error, info, warn}; use super::builder_client::BuilderError; use super::builder_flow::Builder; -use super::proposal::{refuse_future_slot, require_fulu_slot}; +use super::proposal::{ + refuse_finalized_slot, refuse_future_slot, refuse_parent_behind_finality, require_fulu_slot, +}; use super::{ApiError, BodyEncoding}; /// The 500's message when the builder answers a 2xx other than 202: not a @@ -107,8 +109,10 @@ pub(crate) fn finalized_slot(store: &Store) -> Slot { } /// Checked like `post_block`'s fulu path, before anything leaves this node: -/// the parent is held, the block is after it, its proposer is the one the -/// parent's state names for the slot, and the proposer's signature verifies. +/// the block is after the finalized slot, the parent is held and can descend +/// from the finalized checkpoint, the block is after it, its proposer is the +/// one the parent's state names for the slot, and the proposer's signature +/// verifies. /// Then the slot is claimed, and the block goes to the builder. /// /// The submission runs on its own task. Once the proposer has signed it is @@ -151,13 +155,18 @@ pub(crate) async fn post_blinded_block( "blinded blocks are accepted for fulu slots only", ) .and_then(|()| refuse_future_slot(&store.config(), slot)) + .and_then(|()| refuse_finalized_slot(&store, slot)) { return err.into_response(); } - let Some(parent_state) = store.get_state(&signed.message.parent_root).ok().flatten() else { + let parent_root = signed.message.parent_root; + let Some(parent_state) = store.get_state(&parent_root).ok().flatten() else { return ApiError::BadRequest("the block's parent is not known to this node") .into_response(); }; + if let Err(err) = refuse_parent_behind_finality(&store, parent_root, parent_state.slot()) { + return err.into_response(); + } let config = store.config(); let checked = tokio::task::spawn_blocking(move || { if slot <= parent_state.slot() { diff --git a/crates/net/rpc/src/beacon/builder_client/mod.rs b/crates/net/rpc/src/beacon/builder_client/mod.rs index 982b6d8bc..fd8ecc5b9 100644 --- a/crates/net/rpc/src/beacon/builder_client/mod.rs +++ b/crates/net/rpc/src/beacon/builder_client/mod.rs @@ -467,26 +467,37 @@ pub(super) async fn read_body( Ok(body) } -/// `text` on one line, its line breaks written out as `\n` and `\r`, then cut -/// to at most [`MAX_ERROR_TEXT`] bytes at a char boundary. +/// `text` with every character that could break or restyle a line written +/// out, cut to at most [`MAX_ERROR_TEXT`] bytes. /// /// A builder's text reaches this node's logs and a validator client's error -/// message, where a raw line break would start what reads as a log line of -/// its own. +/// message. There a raw line break would start what reads as a log line of +/// its own, and an ESC would start an ANSI sequence that recolors, moves the +/// cursor in or clears a terminal. So every control character +/// (`char::is_control`: C0, DEL and C1) and the Unicode line and paragraph +/// separators U+2028/U+2029, which some viewers break lines at, are written +/// as Rust escapes: `\n`, `\r` and `\t`, and `\u{1b}` for the rest. Everything +/// else passes through unchanged, quotes and backslashes included. +/// +/// The cap is applied to the escaped text, and never cuts a character or an +/// escape in two. pub(super) fn error_text(text: String) -> String { - let mut text = if text.contains(['\n', '\r']) { - text.replace('\n', "\\n").replace('\r', "\\r") - } else { - text - }; - if text.len() > MAX_ERROR_TEXT { - let mut end = MAX_ERROR_TEXT; - while !text.is_char_boundary(end) { - end -= 1; + let mut written = String::with_capacity(text.len().min(MAX_ERROR_TEXT)); + for c in text.chars() { + if c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') { + let escaped = c.escape_default(); + if written.len() + escaped.len() > MAX_ERROR_TEXT { + break; + } + written.extend(escaped); + } else { + if written.len() + c.len_utf8() > MAX_ERROR_TEXT { + break; + } + written.push(c); } - text.truncate(end); } - text + written } /// The fork `version` names, if its bids have the shape [`SignedBuilderBid`] diff --git a/crates/net/rpc/src/beacon/builder_client/tests.rs b/crates/net/rpc/src/beacon/builder_client/tests.rs index a8570a411..b5c26e94a 100644 --- a/crates/net/rpc/src/beacon/builder_client/tests.rs +++ b/crates/net/rpc/src/beacon/builder_client/tests.rs @@ -503,6 +503,47 @@ async fn line_breaks_in_an_error_are_written_out() { assert_eq!(message.len(), 1024); } +/// Every other control character is written out too (an ESC would start an +/// ANSI sequence in a terminal), and so are the Unicode line and paragraph +/// separators, which `char::is_control` does not cover. Quotes and +/// backslashes pass through. +#[tokio::test] +async fn control_characters_in_an_error_are_written_out() { + let relay = MockRelay::start(GENESIS_FORK_VERSION).await; + let body = "\x1b[31mred\x1b[0m\ttab\x07bell\u{85}nel\u{2028}ls\u{2029}ps \"q\" \\ ok"; + relay.answer_headers_with(StatusCode::BAD_REQUEST, TEXT, body.into(), None); + let Err(BuilderError::BadRequest(message)) = get_header(&client_for(&relay)).await else { + panic!("expected a bad request"); + }; + assert_eq!( + message, + r#"\u{1b}[31mred\u{1b}[0m\ttab\u{7}bell\u{85}nel\u{2028}ls\u{2029}ps "q" \ ok"# + ); + + // The same escapes reach a JSON error's message, where the builder can + // send them as `\u001b`. + let relay = MockRelay::start(GENESIS_FORK_VERSION).await; + let body = br#"{"code":400,"message":"\u001b[2Jcleared"}"#.to_vec(); + relay.answer_headers_with(StatusCode::BAD_REQUEST, JSON, body, None); + let Err(BuilderError::BadRequest(message)) = get_header(&client_for(&relay)).await else { + panic!("expected a bad request"); + }; + assert_eq!(message, r"\u{1b}[2Jcleared"); +} + +/// The cap never leaves half an escape behind. +#[test] +fn the_cap_never_cuts_an_escape_in_two() { + // 1020 bytes, then a six-byte escape that would end past the cap. + let text = format!("{}\x1b", "x".repeat(1020)); + assert_eq!(super::error_text(text), "x".repeat(1020)); + let text = format!("{}\x1b", "x".repeat(1018)); + assert_eq!( + super::error_text(text), + format!("{}\\u{{1b}}", "x".repeat(1018)) + ); +} + #[tokio::test] async fn a_server_error_is_an_unexpected_status() { let relay = MockRelay::start(GENESIS_FORK_VERSION).await; diff --git a/crates/net/rpc/src/beacon/builder_flow/bid.rs b/crates/net/rpc/src/beacon/builder_flow/bid.rs index 4c16765e1..ab202f263 100644 --- a/crates/net/rpc/src/beacon/builder_flow/bid.rs +++ b/crates/net/rpc/src/beacon/builder_flow/bid.rs @@ -9,6 +9,23 @@ //! execution client could verify, and the fee recipient and the payment to the //! proposer, see [`validate`]) is trusted to the relay, as every client does: //! the payload is revealed only after the proposer signs. +//! +//! The bid's signature is checked against the key the bid itself names, so it +//! proves the bid arrived as some key signed it, not that a builder the +//! operator chose sent it. That catches a bid corrupted on the way and an +//! honest endpoint's mistakes. It does not stop the endpoint itself, or a +//! man in the middle on a plain-HTTP `--builder`, from changing a bid and +//! signing it again with a key of its own. Nothing here pins the builder's +//! key, and no client reviewed does by default: Lighthouse +//! (`SignedBuilderBid::verify_signature`, `types/src/builder/builder_bid.rs:196`), +//! Teku (`BuilderBidValidatorImpl.java:58`), Prysm +//! (`validateBuilderSignature`, `proposer_bellatrix.go:321`) and Nimbus +//! (`block_payloads.nim:649`) all verify against the bid's own `pubkey`, and +//! Lodestar (`execution/builder/http.ts:186`) does not verify it at all. The +//! endpoint is trusted, as MEV-Boost is: one that lies can win the comparison +//! with a value it never pays or withhold the payload, and either way costs +//! the proposer the slot's reward, though the checks above hold whoever +//! signed the bid. use ethlambda_state_transition::beacon::block_production::PayloadInputs; use ethlambda_state_transition::beacon::bls; @@ -69,6 +86,10 @@ impl Rejection { /// Whether `bid` is acceptable for the proposal `context` describes. /// Cheap checks first; the signature, the one that costs a pairing, last. /// +/// The signature is verified against `bid.message.pubkey`, the key the bid +/// names, so it is an integrity check against an endpoint that is not +/// adversarial, not an authentication of the builder (see the module doc). +/// /// The bid's fork label is not checked here: an electra bid at a fulu slot is /// accepted on purpose (the containers are identical, so a lagging relay's bid /// stays usable), and `BuilderClient` already refuses every other fork. Nor is @@ -125,6 +146,9 @@ pub(crate) fn validate(bid: &SignedBuilderBid, context: &BidContext) -> Result<( // signs. The relay checks it, when it validates the builder's submission // on an execution node before serving the bid. See // `docs/spec_deviations.md`. + // + // Against the key the bid names: whoever can rewrite the bid can sign it + // again under a key of its own (see the module doc). let domain = compute_builder_domain(context.genesis_fork_version); let root = compute_signing_root(message.hash_tree_root(), domain); if !bls::verify(&message.pubkey, root, &bid.signature) { diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index 64a1f87b8..df520af49 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -75,7 +75,7 @@ use libssz_derive::{SszDecode, SszEncode}; use libssz_types::SszList; use serde::Deserialize; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tracing::{debug, error, info, warn}; use crate::beacon::bid_selection::wei_u128; @@ -141,13 +141,14 @@ pub(crate) fn routes() -> Router { /// `SignedBeaconBlock` (gloas), as SSZ or JSON by `Content-Type`. /// /// Checked before it goes anywhere: the fork is fulu (both the header and the -/// slot's place in the schedule), its parent is a block this node holds (not -/// necessarily the head), it is after that parent, its proposer is the one the -/// parent's state advanced to its slot names, the proposer's signature -/// verifies against that state, and its blobs and cell proofs match the -/// block's commitments and verify. Then handed to P2P with the data column -/// sidecars built from them, -/// which gossips both and gives the block to the chain actor to import. The +/// slot's place in the schedule), the slot is after the finalized one, its +/// parent is a block this node holds (not necessarily the head) that can +/// descend from the finalized checkpoint, it is after that parent, its +/// proposer is the one the parent's state advanced to its slot names, the +/// proposer's signature verifies against that state, and its blobs and cell +/// proofs match the block's commitments and verify. Then handed to P2P with +/// the data column sidecars built from them, which gossips both and gives +/// the block to the chain actor to import. The /// full import runs there, so `200` here means validated and broadcast, the /// `gossip` level of `broadcast_validation`, which is the endpoint's default. async fn post_block( @@ -176,12 +177,14 @@ async fn post_block( }; // Before the parent's state is advanced: a fulu body at a gloas slot would // run `upgrade_to_gloas` only to be refused afterwards. + let slot = contents.signed_block.message.slot; if let Err(err) = require_fulu_slot( &store.config(), - contents.signed_block.message.slot, + slot, "blocks are accepted for fulu slots only", ) - .and_then(|()| refuse_future_slot(&store.config(), contents.signed_block.message.slot)) + .and_then(|()| refuse_future_slot(&store.config(), slot)) + .and_then(|()| refuse_finalized_slot(&store, slot)) { return err.into_response(); } @@ -191,14 +194,14 @@ async fn post_block( kzg_proofs, blobs, } = contents; - let Some(parent_state) = store - .get_state(&signed_block.message.parent_root) - .ok() - .flatten() - else { + let parent_root = signed_block.message.parent_root; + let Some(parent_state) = store.get_state(&parent_root).ok().flatten() else { return ApiError::BadRequest("the block's parent is not known to this node") .into_response(); }; + if let Err(err) = refuse_parent_behind_finality(&store, parent_root, parent_state.slot()) { + return err.into_response(); + } let config = store.config(); let checked = tokio::task::spawn_blocking(move || { let slot = signed_block.message.slot; @@ -280,8 +283,9 @@ async fn post_block( /// gloas block carries no payload and no blobs of its own (the envelope and /// the data columns follow through `publishExecutionPayloadEnvelope`). /// -/// Checked as the fulu one is, before it goes anywhere: the parent is held, -/// the slot is after the parent's and scheduled at gloas, the proposer is the +/// Checked as the fulu one is, before it goes anywhere: the parent is held +/// and can descend from the finalized checkpoint, the slot is after the +/// finalized one and the parent's and scheduled at gloas, the proposer is the /// slot's, and the proposer's signature verifies against the parent's state /// advanced to the slot. The advance and the signature are CPU-bound and run /// off the runtime. @@ -303,6 +307,7 @@ async fn post_gloas_block( "blocks are accepted for gloas slots only", ) .and_then(|()| refuse_future_slot(&config, block.slot())) + .and_then(|()| refuse_finalized_slot(store, block.slot())) { return err.into_response(); } @@ -311,6 +316,10 @@ async fn post_gloas_block( Ok(None) => return ApiError::BadRequest("the block's parent is not held").into_response(), Err(_) => return ApiError::Internal("store read failed").into_response(), }; + if let Err(err) = refuse_parent_behind_finality(store, block.parent_root(), parent_state.slot()) + { + return err.into_response(); + } if block.slot() <= parent_state.slot() { return ApiError::BadRequest("the block is not after its parent").into_response(); } @@ -463,6 +472,51 @@ pub(super) fn refuse_future_slot(config: &Config, slot: Slot) -> Result<(), ApiE Ok(()) } +/// Refuses a published block at or below the finalized checkpoint's slot, +/// before its parent's state is loaded: gossip's `[IGNORE] block.slot > +/// compute_start_slot_at_epoch(finalized_checkpoint.epoch)`, which every peer +/// would apply to it anyway. +/// +/// With [`refuse_parent_behind_finality`] this bounds how far a publish +/// endpoint advances a parent's state, which it does before the proposer's +/// signature is checked, so for any caller. [`refuse_future_slot`] caps the +/// block's slot; these cap how far back its parent may be. +pub(super) fn refuse_finalized_slot(store: &Store, slot: Slot) -> Result<(), ApiError> { + if slot <= finalized_slot(store) { + return Err(ApiError::BadRequest( + "the block is not after the finalized slot", + )); + } + Ok(()) +} + +/// Refuses a parent the finalized checkpoint cannot be an ancestor of, before +/// its state is advanced: gossip's `[REJECT]` for a block whose chain does not +/// pass through the finalized checkpoint, for the parents it can be decided +/// for from the parent alone. +/// +/// A parent at or below the finalized checkpoint's slot is its own +/// checkpoint block (`get_checkpoint_block` stops at it), so it descends from +/// the finalized checkpoint only by being its block. The finalized block may +/// itself sit below that slot, when the epoch's first slots were empty, which +/// is why the root is compared rather than the slot alone. A parent above the +/// slot is not walked back here; gossip validation and import walk it, and +/// the advance from such a parent is bounded by the finalized slot below and +/// the wall clock above. +pub(super) fn refuse_parent_behind_finality( + store: &Store, + parent_root: H256, + parent_slot: Slot, +) -> Result<(), ApiError> { + let finalized_root = store.beacon_finalized_checkpoint().root; + if parent_slot <= finalized_slot(store) && parent_root != finalized_root { + return Err(ApiError::BadRequest( + "the block's parent does not descend from the finalized checkpoint", + )); + } + Ok(()) +} + #[allow(clippy::too_many_arguments)] async fn get_block( Path(slot): Path, @@ -614,8 +668,9 @@ struct BuilderRequest<'a> { /// circuit breaker quiet, the builder's bid is fetched alongside the local /// payload. Both are checked without a state transition, [`choice::choose`] /// picks one, and only that one is built into a block; the other is built only -/// if the first fails. Without a builder the block is built exactly as it is -/// without one. +/// if the first fails. A valid bid waits for the local payload only until +/// [`LOCAL_BUILD_DEADLINE`] (see [`local_and_bid`]). Without a builder the +/// block is built exactly as it is without one. #[allow(clippy::too_many_arguments)] async fn produce( store: &Store, @@ -649,18 +704,20 @@ async fn produce( // Asked alongside the build rather than before it, so a healthy execution // client costs the block nothing for its version and a silent one costs it // at most the version's own timeout. The builder's bid is asked alongside - // as well, bounded by its own header timeout. - let (built, el_version, bid) = tokio::join!( - build_payload(store, engine, fee_recipients, &state, head_root, proposer), + // as well, bounded by its own header timeout, and checked as soon as it + // arrives, so a valid one can bound the wait for the local payload. + let local = build_payload(store, engine, fee_recipients, &state, head_root, proposer); + let bid = async { + let request = request.as_ref()?; + let bid = fetch_bid(request, slot).await?; + accepted_bid(&state, &config, request, bid) + }; + let ((built, bid), el_version) = tokio::join!( + local_and_bid(local, bid, slot), graffiti::execution_client_version(engine, own_version), - fetch_bid(request.as_ref(), slot), ); let graffiti = graffiti::with_client_versions(graffiti, el_version.as_ref(), own_version); let local = built.and_then(|built| local_parts(built, slot)); - let bid = request - .as_ref() - .zip(bid) - .and_then(|(request, bid)| accepted_bid(&state, &config, request, bid)); // Nothing to build a block from: skip the packing, whose sync aggregate // costs a BLS verification. let (local, mut bid) = match (local, bid) { @@ -955,9 +1012,69 @@ fn builder_request<'a>( }) } -/// The builder's bid, or `None` when it was not asked, had none, or failed. -async fn fetch_bid(request: Option<&BuilderRequest<'_>>, slot: Slot) -> Option { - let request = request?; +/// How long, from when the payload and the bid are asked for, the local +/// payload may keep a validated builder bid waiting. +/// +/// Without a bound the wait is the execution client's own: `getPayload` is one +/// attempt at 1 s, but `forkchoiceUpdated` is up to three attempts at +/// `ENGINE_TIMEOUT` (8 s) with backoff, so a stalled execution client would +/// hold a valid bid past the 12 s slot and the proposal would be missed. +/// +/// Two seconds follows the other clients. Lodestar waits for both sources +/// until 1667 bps of the slot (2 s of 12 s, `BLOCK_PRODUCTION_RACE_CUTOFF_BPS`, +/// `api/impl/validator/index.ts:129`) and then takes whichever has answered +/// (`resolveOrRacePromises`, `utils/src/promise.ts:100`), measuring from the +/// slot's start where this measures from the request, the same instant for a +/// validator client that asks at the slot boundary, as they do. Lighthouse +/// waits for both (`execution_layer/src/lib.rs:1077`) with no deadline of its +/// own, but makes each engine call once (`Engine::request`, `engines.rs:352`), +/// `getPayload` at 2 s and `forkchoiceUpdated` at 8 s +/// (`engine_api/http.rs:52,59`); it usually starts the build before the slot, +/// so only `getPayload` is on the path. A healthy execution client answers in +/// well under a second, so it is never cut off. +const LOCAL_BUILD_DEADLINE: Duration = Duration::from_secs(2); + +/// The local payload and the accepted builder bid, both asked for at once. +/// +/// Once a bid has passed its checks, the local build has until +/// [`LOCAL_BUILD_DEADLINE`] after this was called; past it the build is +/// dropped and answered as failed, so the bid is served through the same +/// fallback as for any failed local build. Without a bid there is nothing to +/// fall back on, so the local build is waited for exactly as it is without a +/// builder: giving up on it then could only turn a late block into none. +async fn local_and_bid( + local: impl Future>, + bid: impl Future>, + slot: Slot, +) -> (Result, Option) { + let deadline = tokio::time::Instant::now() + LOCAL_BUILD_DEADLINE; + tokio::pin!(local, bid); + let bid = tokio::select! { + built = &mut local => return (built, bid.await), + bid = &mut bid => bid, + }; + if bid.is_none() { + return (local.await, None); + } + match tokio::time::timeout_at(deadline, local).await { + Ok(built) => (built, bid), + Err(_) => { + let deadline_ms = LOCAL_BUILD_DEADLINE.as_millis() as u64; + warn!( + %slot, + deadline_ms, + "The execution client did not return a payload in time; the builder's bid stands in" + ); + let err = ApiError::ServiceUnavailable( + "the execution client did not return a payload in time", + ); + (Err(err), bid) + } + } +} + +/// The builder's bid, or `None` when it had none or failed. +async fn fetch_bid(request: &BuilderRequest<'_>, slot: Slot) -> Option { request .builder .client @@ -1463,10 +1580,13 @@ mod tests { let parent = parent_state(); let state = state_to_build_on(); let produced = produce(&state, true, Vec::new()).unwrap(); - let (mut store, _anchor) = beacon_store_with_config(parent.clone(), config()); + let parent_slot = parent.slot(); + let (mut store, anchor) = beacon_store_with_config(parent.clone(), config()); store .insert_state(produced.block.parent_root, parent) .unwrap(); + let parent_root = produced.block.parent_root; + crate::test_utils::finalize_at(&mut store, anchor, parent_root, parent_slot); let signed = containers::gloas::SignedBeaconBlock { signature: sign_block(&state, &produced.block), @@ -1538,10 +1658,13 @@ mod tests { let parent = parent_state(); let state = state_to_build_on(); let produced = produce(&state, true, Vec::new()).unwrap(); - let (mut store, _anchor) = beacon_store_with_config(parent.clone(), config()); + let parent_slot = parent.slot(); + let (mut store, anchor) = beacon_store_with_config(parent.clone(), config()); store .insert_state(produced.block.parent_root, parent) .unwrap(); + let parent_root = produced.block.parent_root; + crate::test_utils::finalize_at(&mut store, anchor, parent_root, parent_slot); let sign = |block: &containers::gloas::BeaconBlock| containers::gloas::SignedBeaconBlock { signature: sign_block(&state, block), message: block.clone(), @@ -1789,6 +1912,67 @@ mod tests { .unwrap() } + /// An execution client that takes every request and never answers. + async fn silent_engine() -> EngineClient { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let router = Router::new().route( + "/", + post(|| async { std::future::pending::().await }), + ); + tokio::spawn(async move { axum::serve(listener, router).await }); + EngineClient::new( + format!("http://{address}"), + ethlambda_engine::JwtSecret::new([0x0f; 32]), + ) + .unwrap() + } + + /// A stalled execution client holds a valid bid back only until the + /// local build's deadline, not for its own retry ladder (one + /// `forkchoiceUpdated` attempt alone is `ENGINE_TIMEOUT`). + #[tokio::test] + async fn a_silent_execution_client_gives_way_to_a_valid_bid_at_the_deadline() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |_| {}); + let started = std::time::Instant::now(); + let answer = produce_answer(&world, Some(silent_engine().await), None, "").await; + let elapsed = started.elapsed(); + answer.assert(&world, true, Ð.to_string(), 1); + assert!(elapsed >= LOCAL_BUILD_DEADLINE, "{elapsed:?}"); + assert!( + elapsed < ethlambda_engine::client::ENGINE_TIMEOUT, + "{elapsed:?}" + ); + } + + /// Without a bid to fall back on, a slow local build is waited for past + /// the deadline, as it is without a builder. + #[tokio::test] + async fn without_a_bid_a_slow_local_build_is_waited_for() { + let world = builder_world(true, SyncStatus::Synced).await; + let delay = LOCAL_BUILD_DEADLINE + Duration::from_millis(500); + let engine = + super::super::validator_client_tests::fake_execution_client_slow_to_build(delay).await; + let started = std::time::Instant::now(); + let answer = produce_answer(&world, Some(engine), None, "").await; + answer.assert(&world, false, "42", 1); + assert!(started.elapsed() >= delay, "{:?}", started.elapsed()); + } + + /// A local build that beats the deadline is weighed against the bid as + /// always, however late in the window it answers. + #[tokio::test] + async fn a_local_build_inside_the_deadline_still_competes() { + let world = builder_world(true, SyncStatus::Synced).await; + world.offer(ETH, |_| {}); + let delay = LOCAL_BUILD_DEADLINE / 2; + let engine = + super::super::validator_client_tests::fake_execution_client_slow_to_build(delay).await; + let answer = produce_answer(&world, Some(engine), None, "&builder_boost_factor=0").await; + answer.assert(&world, false, "42", 1); + } + #[tokio::test] async fn a_bid_worth_more_than_the_local_payload_is_served_blinded() { let world = builder_world(true, SyncStatus::Synced).await; @@ -2170,6 +2354,146 @@ mod tests { const TAKEN: &str = "this node already published a different block for the slot"; + const NOT_AFTER_FINALIZED: &str = "the block is not after the finalized slot"; + const NOT_FROM_FINALIZED: &str = + "the block's parent does not descend from the finalized checkpoint"; + + /// A copy of the head state held under a made-up root. The test stores + /// finalize their head at its own slot, so this is a parent at the + /// finalized slot that is not the finalized block: one the finalized + /// checkpoint cannot be an ancestor of. + fn hold_parent_beside_finality(store: &mut Store) -> H256 { + let (head_root, head_state) = super::super::validator::head(store).unwrap(); + assert_eq!(store.beacon_finalized_checkpoint().root, head_root); + assert_eq!(head_state.slot(), finalized_slot(store)); + let root = H256::repeat_byte(7); + store.insert_state(root, (*head_state).clone()).unwrap(); + root + } + + /// An unsigned fulu block with no blobs. + fn fulu_contents( + slot: Slot, + parent_root: H256, + proposer_index: ValidatorIndex, + ) -> FuluSignedBlockContents { + FuluSignedBlockContents { + signed_block: SignedBeaconBlock { + message: BeaconBlock { + slot, + proposer_index, + parent_root, + state_root: H256::ZERO, + body: containers::electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }, + kzg_proofs: Default::default(), + blobs: Default::default(), + } + } + + /// A block at the finalized slot, and one after it on a parent the + /// finalized checkpoint is not an ancestor of, are refused before the + /// parent's state is advanced, so before the signature is looked at. + #[tokio::test] + async fn a_fulu_block_behind_finality_is_refused_before_advancing() { + let (mut store, head_slot) = fulu_store(); + let (head_root, _) = super::super::validator::head(&store).unwrap(); + let stale = hold_parent_beside_finality(&mut store); + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let app = routes().with_state(store).layer(Extension(p2p)); + for (slot, parent_root, message) in [ + (head_slot, head_root, NOT_AFTER_FINALIZED), + (head_slot + 1, stale, NOT_FROM_FINALIZED), + ] { + let contents = fulu_contents(slot, parent_root, 0); + let (status, json) = post_full(app.clone(), &contents).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], message); + } + assert!(network.blocks.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_gloas_block_behind_finality_is_refused_before_advancing() { + let (mut store, head_slot) = gloas_scheduled_store(); + let (head_root, _) = super::super::validator::head(&store).unwrap(); + let stale = hold_parent_beside_finality(&mut store); + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let app = routes().with_state(store).layer(Extension(p2p)); + for (slot, parent_root, message) in [ + (head_slot, head_root, NOT_AFTER_FINALIZED), + (head_slot + 1, stale, NOT_FROM_FINALIZED), + ] { + let containers::SignedBeaconBlock::Gloas(block) = + crate::test_utils::gloas_beacon_block(slot, parent_root, H256::ZERO, H256::ZERO) + else { + unreachable!("built as gloas") + }; + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "gloas") + .header(header::CONTENT_TYPE, crate::SSZ_CONTENT_TYPE) + .body(Body::from(block.to_ssz())) + .unwrap(); + let (status, json) = respond(app.clone(), request).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], message); + } + assert!(network.blocks.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_blinded_block_behind_finality_never_reaches_the_builder() { + let mut world = builder_world(true, SyncStatus::Synced).await; + let stale = hold_parent_beside_finality(&mut world.store); + let mut at_finalized = signed_blinded(&world, 1, None); + at_finalized.message.slot = world.slot - 1; + let mut on_stale = signed_blinded(&world, 1, None); + on_stale.message.parent_root = stale; + for (block, message) in [ + (at_finalized, NOT_AFTER_FINALIZED), + (on_stale, NOT_FROM_FINALIZED), + ] { + let (status, json) = post_blinded(publish_app(&world), &block).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], message); + } + assert!(world.relay.blinded_blocks().is_empty()); + } + + /// The finalized block sits below its epoch's first slot when that slot + /// was empty. A block on it descends from the finalized checkpoint, so it + /// passes both finality checks and is judged on its signature. + #[tokio::test] + async fn a_block_on_a_finalized_block_below_its_epoch_start_passes_the_finality_checks() { + use ethlambda_storage::ForkCheckpoints; + use ethlambda_types::checkpoint::Checkpoint; + + let (mut store, head_slot) = fulu_store(); + let (head_root, head_state) = super::super::validator::head(&store).unwrap(); + let next_epoch_start = head_slot + preset::SLOTS_PER_EPOCH; + let finalized = Checkpoint { + root: head_root, + slot: next_epoch_start, + }; + let checkpoints = ForkCheckpoints::new(head_root, None, Some(finalized)); + store.update_checkpoints(checkpoints).unwrap(); + assert_eq!(finalized_slot(&store), next_epoch_start); + + let slot = next_epoch_start + 1; + let state = advance_to_slot(&head_state, slot, &store.config()).unwrap(); + let proposer = get_beacon_proposer_index(&state).unwrap(); + let network: RpcToP2PRef = Arc::new(RecordingNetwork::default()); + let app = routes().with_state(store).layer(Extension(network)); + let contents = fulu_contents(slot, head_root, proposer); + let (status, json) = post_full(app, &contents).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + assert_eq!(json["message"], "invalid block signature"); + } + #[tokio::test] async fn a_block_published_in_full_blocks_a_different_blinded_one() { let world = builder_world(true, SyncStatus::Synced).await; diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index aa1a665b8..acb184e39 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -127,16 +127,22 @@ async fn serve_at( // A block built on this head names the head block's real root as its // parent, which the fixture's placeholder anchor block does not have: // keep the state under that root too, as a real anchor would be. + let parent_root = real_head_root(&state); + store.insert_state(parent_root, state.clone()).unwrap(); + let (client, network, _pool, address) = spawn_server(store.clone(), engine).await; + (client, state, network, address, store) +} + +/// The root a block built on `state` names as its parent: its latest block +/// header's, which the fixtures' placeholder anchor block does not have. +fn real_head_root(state: &BeaconState) -> ethlambda_types::primitives::H256 { let advanced = ethlambda_state_transition::beacon::block_production::advance_to_slot( - &state, + state, state.slot() + 1, ðlambda_types::beacon::config::Config::mainnet(), ) .unwrap(); - let parent_root = advanced.latest_block_header().hash_tree_root(); - store.insert_state(parent_root, state.clone()).unwrap(); - let (client, network, _pool, address) = spawn_server(store.clone(), engine).await; - (client, state, network, address, store) + advanced.latest_block_header().hash_tree_root() } /// Serve `store` through the router the real server builds, over a socket on an @@ -441,7 +447,8 @@ fn valid_blob(seed: u8) -> Vec { /// The fulu bundle carries `blob_count` valid blobs with their commitments and /// cell proofs (flattened blob by blob, as `BlobsBundleV2` does). pub(super) async fn fake_execution_client(blob_count: usize) -> ethlambda_engine::EngineClient { - fake_execution_client_with(blob_count, None, "0x2a", false).await + let delay = std::time::Duration::ZERO; + fake_execution_client_with(blob_count, None, "0x2a", false, delay).await } /// [`fake_execution_client`] with no blobs, whose fulu payload pays @@ -450,13 +457,23 @@ pub(super) async fn fake_execution_client_paying( block_value: &'static str, should_override_builder: bool, ) -> ethlambda_engine::EngineClient { - fake_execution_client_with(0, None, block_value, should_override_builder).await + let delay = std::time::Duration::ZERO; + fake_execution_client_with(0, None, block_value, should_override_builder, delay).await +} + +/// [`fake_execution_client`] with no blobs, holding every `forkchoiceUpdated` +/// answer back for `delay`: an execution client slow to start a build. +pub(super) async fn fake_execution_client_slow_to_build( + delay: std::time::Duration, +) -> ethlambda_engine::EngineClient { + fake_execution_client_with(0, None, "0x2a", false, delay).await } /// [`fake_execution_client`] whose gloas payloads carry `blob`, with its real /// commitment and cell proofs, in the blobs bundle. async fn fake_execution_client_with_blob(blob: Option>) -> ethlambda_engine::EngineClient { - fake_execution_client_with(0, blob, "0x2a", false).await + let delay = std::time::Duration::ZERO; + fake_execution_client_with(0, blob, "0x2a", false, delay).await } async fn fake_execution_client_with( @@ -464,6 +481,7 @@ async fn fake_execution_client_with( blob: Option>, block_value: &'static str, should_override_builder: bool, + forkchoice_delay: std::time::Duration, ) -> ethlambda_engine::EngineClient { use axum::{Json, routing::post}; use ethlambda_state_transition::beacon::kzg; @@ -529,7 +547,11 @@ async fn fake_execution_client_with( "excessBlobGas": "0x0", }) }; - let result = match request["method"].as_str() { + let method = request["method"].as_str(); + if let Some("engine_forkchoiceUpdatedV3" | "engine_forkchoiceUpdatedV4") = method { + tokio::time::sleep(forkchoice_delay).await; + } + let result = match method { Some("engine_forkchoiceUpdatedV3" | "engine_forkchoiceUpdatedV4") => { let mut attributes = request["params"][1].clone(); attributes["parentHash"] = request["params"][0]["headBlockHash"].clone(); @@ -605,8 +627,15 @@ async fn produce_with_pool( }; use ethlambda_types::beacon::constants::{DOMAIN_BEACON_PROPOSER, DOMAIN_RANDAO}; - let (client, state, network, address, store) = + let (client, state, network, address, mut store) = serve_at(Some(fake_execution_client(blob_count).await)).await; + // The publish endpoints refuse a parent at the finalized slot other than + // the finalized block, so finalize the root the block names, as a real + // anchor's checkpoint does. Here only: the vote paths walk the block + // index, which holds the placeholder anchor. + let (anchor_root, _) = crate::beacon::validator::head(&store).unwrap(); + let parent_root = real_head_root(&state); + crate::test_utils::finalize_at(&mut store, anchor_root, parent_root, state.slot()); fill_pool(&store, &state); let slot = state.slot() + 1; let advanced = advance_to_slot( diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index 42dc58a1f..dcc8ba273 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -831,6 +831,20 @@ pub(crate) mod test_utils { (store, root) } + /// Finalizes `root` at `slot`'s epoch, leaving the head at `head`. + /// + /// For a store that keeps its anchor state under a second root, the one a + /// block built on it names as its parent: a real anchor's finalized + /// checkpoint names that root, and the publish endpoints refuse a parent + /// at the finalized slot that is not the finalized block. + pub(crate) fn finalize_at(store: &mut Store, head: H256, root: H256, slot: u64) { + let finalized = Checkpoint { root, slot }; + let checkpoints = ForkCheckpoints::new(head, None, Some(finalized)); + store + .update_checkpoints(checkpoints) + .expect("finalize the anchor's second root"); + } + /// A beacon store whose head is `block` (a block at `state`'s slot) with /// `state` as its post-state, on a clock whose slot `clock_slot` began a /// second ago. diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index 1aa13b184..c0cec5c4f 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -322,6 +322,14 @@ An RPC *error* is not retried: that is the execution client answering, with a refusal but an answer, so asking again would get the same refusal and burn the ladder for nothing. +`engine_getPayloadV5` and `engine_getPayloadV6` have no ladder: one attempt at +`ENGINE_GET_PAYLOAD_TIMEOUT` (1 s). execution-apis #958 removed the method's +timeout ("MUST respond immediately", and the consensus client "MAY abort such a +call at its own discretion"); 1 s is the value it replaced and what Nimbus and +Prysm (before gloas) use, where Lighthouse and Teku allow 2 s. A retry could not +deliver the slot's payload in time, and an execution client may stop the build +once it has served the call. + After the last attempt the block is **dropped**. `optimistic-sync.md` requires exactly that much ("a consensus engine MUST NOT import the block and MUST NOT apply it to the fork choice store"), but nothing here re-drives the call diff --git a/docs/rpc.md b/docs/rpc.md index c6063c144..1712a24a4 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -472,11 +472,22 @@ the chain actor writes, so no request waits on the actor. is too far behind, or too many slots were skipped (the `--builder-fallback-*` flags set the thresholds). The builder's `getHeader` is then asked concurrently with the local build, bounded by its own timeout, and an error - or a timeout means no bid. A bid is checked before it competes: the head's - execution block as its parent, the proposal's `prev_randao` + or a timeout means no bid. Once a bid has passed its checks, the local build + has until two seconds after the two were asked for (`LOCAL_BUILD_DEADLINE`; + Lodestar's race cutoff and Lighthouse's `getPayload` timeout are both 2 s): + an execution client still silent then is given up on with a warning, and + the bid is served as for any failed local build. Without a valid bid the + local build is waited for as it is without a builder. A bid is checked + before it competes: the head's execution block as its parent, the + proposal's `prev_randao` and timestamp, the parent's block number plus one, the withdrawals root, the blob commitment and blob gas bounds, a gas limit within one step of the parent's toward the registered target, - and the builder's signature. Its fee recipient is not checked, since builders + and its signature, verified against the key the bid itself names. That + catches a corrupted bid and an honest endpoint's mistakes, not the endpoint + or a man in the middle on a plain-HTTP `--builder` signing a changed bid + again under a key of its own: the endpoint is trusted, as in every client + (Lighthouse, Teku, Prysm and Nimbus verify against the bid's own key too, + and Lodestar does not verify it). Its fee recipient is not checked, since builders name themselves there and pay the proposer with a transaction this node never sees before signing; the relay checks that payment (see `spec_deviations.md`). A bid that passes goes through `assemble_blinded_block` @@ -511,9 +522,14 @@ the chain actor writes, so no request waits on the actor. content type is a `415`). The JSON is the specification's: lodestar's and nimbus's validator clients post it, teku's falls back to it, and lighthouse and prysm post SSZ. Both reach the same checks after decoding. Fulu: a - `SignedBlockContents`. A slot the schedule does not place at fulu, or one that - has not started yet (see `produceBlockV3`), is a `400`, checked before any - state is advanced. The block is then checked against its + `SignedBlockContents`. A slot the schedule does not place at fulu, one that + has not started yet (see `produceBlockV3`), or one at or below the finalized + checkpoint's first slot is a `400`, and so is a parent at or below that slot + that is not the finalized block itself, since the finalized checkpoint + cannot be its ancestor (gossip's IGNORE and REJECT rules for the two). All + are checked before any state is advanced, and so before the signature: they + bound how far an unauthenticated request can make this node advance an old + parent's state. The block is then checked against its **parent's** state advanced to its slot: an unknown parent, a slot not after the parent's, the wrong proposer or a bad proposer signature is a `400`. Then every blob's cells are computed and all cell proofs batch-verified on a @@ -524,7 +540,8 @@ the chain actor writes, so no request waits on the actor. last the chain actor gets this node's custody columns, then the block, to import. Gloas: a bare `gloas::SignedBeaconBlock` (a gloas block carries no payload or blobs, so no sidecars go with it; they follow in the envelope). The - slot must be scheduled at gloas and have started, the parent must be held, the slot must be + slot must be scheduled at gloas, have started and be after the finalized + slot, the parent must be held and pass the same finality check, the slot must be after the parent's, and the proposer and signature are checked against the parent's state advanced to the slot (all `400` otherwise), before the block is gossiped and handed to the chain actor. This node publishes one block per @@ -535,7 +552,8 @@ the chain actor writes, so no request waits on the actor. - **`POST beacon/blinded_blocks`** takes a fulu `SignedBlindedBeaconBlock` as JSON or SSZ, by `Content-Type` as above, with `Eth-Consensus-Version: fulu`. It is checked as a fulu `beacon/blocks` is, before anything leaves this node: - the slot is scheduled at fulu and has started, the parent is held and before the block, the + the slot is scheduled at fulu, has started and is after the finalized slot, + the parent is held, passes the finality check and is before the block, the proposer is the one the parent's state names for the slot, and the proposer's signature verifies (a blinded block has its full block's root, so the check is the same). Then the slot is claimed, shared with `beacon/blocks`, and the From ebd338aacd98b96a17742e5aa14e5685d19f051a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:07:10 -0300 Subject: [PATCH 19/20] style(rpc): format the merged shown_location Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- crates/net/rpc/src/beacon/builder_client/error.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/net/rpc/src/beacon/builder_client/error.rs b/crates/net/rpc/src/beacon/builder_client/error.rs index 83b3939f0..6215464db 100644 --- a/crates/net/rpc/src/beacon/builder_client/error.rs +++ b/crates/net/rpc/src/beacon/builder_client/error.rs @@ -159,10 +159,7 @@ pub(super) fn shown_target(base: &Url, location: &[u8]) -> String { base.join(&location) .ok() .and_then(without_credentials) - .map_or_else( - || UNSHOWN_LOCATION.to_owned(), - |url| error_text(url.into()), - ) + .map_or_else(|| UNSHOWN_LOCATION.to_owned(), |url| error_text(url.into())) } /// `url` without its user and password, or `None` when it cannot hold them From f4c72d223fafe57e094ec47d067246f8660c661a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:31:35 -0300 Subject: [PATCH 20/20] fix(rpc): refuse block production while syncing, and advance V3's head off the runtime produceBlockV3 and produceBlockV4 now answer 503 while the node is syncing, after the 400 checks and before the head is read, as the duties endpoints do. produceBlockV3 reads the head and advances it to the slot on a blocking task, as produceBlockV4's prepare already did. The gate does not bound a long advance: the sync tracker reads a node more than 8 slots behind as a stalled network rather than as syncing (#559), so the blocking task is what keeps that advance off the async runtime. --- .../rpc/src/beacon/builder_market_tests.rs | 3 + crates/net/rpc/src/beacon/gloas_proposal.rs | 31 +++++++- crates/net/rpc/src/beacon/proposal.rs | 79 +++++++++++++------ docs/rpc.md | 8 +- 4 files changed, 92 insertions(+), 29 deletions(-) diff --git a/crates/net/rpc/src/beacon/builder_market_tests.rs b/crates/net/rpc/src/beacon/builder_market_tests.rs index a14e7e930..545b53ebc 100644 --- a/crates/net/rpc/src/beacon/builder_market_tests.rs +++ b/crates/net/rpc/src/beacon/builder_market_tests.rs @@ -280,6 +280,9 @@ impl World { .with_state(self.store.clone()) .layer(Extension(p2p)) .layer(Extension(engine.map(|engine| engine.client.clone()))) + .layer(Extension(ethlambda_blockchain::SyncStatusController::new( + ethlambda_blockchain::metrics::SyncStatus::Synced, + ))) .layer(Extension(SharedPayloadAttestationPool::default())) .layer(Extension(SharedSyncCommitteePool::default())) .layer(Extension(crate::beacon::graffiti::OwnVersion(Arc::new( diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs index 2eaf9467f..a476bcdf8 100644 --- a/crates/net/rpc/src/beacon/gloas_proposal.rs +++ b/crates/net/rpc/src/beacon/gloas_proposal.rs @@ -44,7 +44,7 @@ use axum::{ response::{IntoResponse, Response}, routing::{get, post}, }; -use ethlambda_blockchain::checkpoint_hash; +use ethlambda_blockchain::{SyncStatusController, checkpoint_hash, metrics::SyncStatus}; use ethlambda_engine::{ CustodyColumns, EngineClient, ForkchoiceStateV1, building::{BuiltGloasPayload, PayloadAttributesV3, PayloadAttributesV4}, @@ -229,6 +229,7 @@ async fn post_produce_block( Extension(custody): Extension, Extension(cache): Extension, Extension(OwnVersion(own_version)): Extension, + Extension(sync_status): Extension, headers: HeaderMap, body: Bytes, ) -> Response { @@ -272,6 +273,10 @@ async fn post_produce_block( if builders > 0 { debug!(%slot, builders, "Not consulting the builder entries of the block production request"); } + // Before the head is read, as `produceBlockV3` does. + if sync_status.get() == SyncStatus::Syncing { + return ApiError::ServiceUnavailable("the node is syncing").into_response(); + } // Without an execution client a block can only be built on a bid, so a node // holding none for the slot has nothing to offer. if engine.is_none() && !market.has_bids_for_slot(slot) { @@ -1204,9 +1209,18 @@ mod tests { } fn produce_app(store: Store, engine: Option) -> Router { + produce_app_with_sync(store, engine, SyncStatus::Synced) + } + + fn produce_app_with_sync( + store: Store, + engine: Option, + sync: SyncStatus, + ) -> Router { routes() .with_state(store) .layer(Extension(engine)) + .layer(Extension(SyncStatusController::new(sync))) .layer(Extension(SharedSyncCommitteePool::default())) .layer(Extension(SharedPayloadAttestationPool::default())) .layer(Extension(SharedBuilderMarket::default())) @@ -1276,6 +1290,21 @@ mod tests { assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); } + /// The syncing refusal precedes the execution-client one, so the message + /// tells the two apart. + #[tokio::test] + async fn producing_on_a_syncing_node_is_a_503() { + let (store, head_slot) = gloas_scheduled_store(); + let app = produce_app_with_sync(store, None, SyncStatus::Syncing); + let (status, json) = respond( + app, + produce_request(head_slot + 1, "&include_payload=true", EMPTY_CONFIG), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(json["message"], "the node is syncing"); + } + #[tokio::test] async fn the_randao_skip_flag_is_refused_with_a_value_or_a_real_reveal() { let (store, head_slot) = gloas_scheduled_store(); diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index df520af49..d8d3e3423 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -26,7 +26,7 @@ use axum::{ response::{IntoResponse, Response}, routing::{get, post}, }; -use ethlambda_blockchain::metrics::SyncStatus; +use ethlambda_blockchain::{SyncStatusController, metrics::SyncStatus}; use ethlambda_engine::{ EngineClient, ForkchoiceStateV1, building::{BuiltPayload, PayloadAttributesV3}, @@ -526,6 +526,7 @@ async fn get_block( Extension(sync_pool): Extension, Extension(fee_recipients): Extension, Extension(OwnVersion(own_version)): Extension, + Extension(sync_status): Extension, builder: Option>>, headers: HeaderMap, ) -> Response { @@ -552,6 +553,12 @@ async fn get_block( { return err.into_response(); } + // Before the head is read: a syncing node's head is not the chain's. This + // does not bound the advance in `produce`: the tracker reads a node more + // than 8 slots behind as a stalled network, not as syncing (#559). + if sync_status.get() == SyncStatus::Syncing { + return ApiError::ServiceUnavailable("the node is syncing").into_response(); + } let Some(engine) = engine else { return ApiError::ServiceUnavailable( "no execution client configured to build a payload with", @@ -671,6 +678,10 @@ struct BuilderRequest<'a> { /// if the first fails. A valid bid waits for the local payload only until /// [`LOCAL_BUILD_DEADLINE`] (see [`local_and_bid`]). Without a builder the /// block is built exactly as it is without one. +/// +/// The head state is advanced to `slot` on a blocking thread, as +/// `produceBlockV4` does: behind a stalled network, the head can be epochs +/// short of the slot, and that advance is the work the proposal needs. #[allow(clippy::too_many_arguments)] async fn produce( store: &Store, @@ -686,14 +697,21 @@ async fn produce( graffiti: Bytes32, ) -> Result<(Produced, String, ForkName), ApiError> { let config = store.config(); - let (head_root, head_state) = head(store)?; - if slot <= head_state.slot() { - return Err(ApiError::BadRequest("slot is not after the head block")); - } - let state = advance_to_slot(&head_state, slot, &config).map_err(|err| { - warn!(%slot, %err, "Advancing the head state to the proposal slot failed"); - ApiError::Internal("advancing the head state failed") - })?; + let advance_store = store.clone(); + let advance_config = Arc::clone(&config); + let (head_root, state) = tokio::task::spawn_blocking(move || { + let (head_root, head_state) = head(&advance_store)?; + if slot <= head_state.slot() { + return Err(ApiError::BadRequest("slot is not after the head block")); + } + let state = advance_to_slot(&head_state, slot, &advance_config).map_err(|err| { + warn!(%slot, %err, "Advancing the head state to the proposal slot failed"); + ApiError::Internal("advancing the head state failed") + })?; + Ok((head_root, state)) + }) + .await + .map_err(|_| ApiError::Internal("advancing the head state failed"))??; let fork = state.fork_name(); let proposer = ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index(&state) @@ -1303,9 +1321,18 @@ mod tests { } fn produce_app(store: Store, engine: Option) -> Router { + produce_app_with_sync(store, engine, SyncStatusController::new(SyncStatus::Synced)) + } + + fn produce_app_with_sync( + store: Store, + engine: Option, + sync_status: SyncStatusController, + ) -> Router { routes() .with_state(store) .layer(Extension(engine)) + .layer(Extension(sync_status)) .layer(Extension(SharedSyncCommitteePool::default())) .layer(Extension(FeeRecipients::default())) .layer(Extension(OwnVersion(Arc::new( @@ -1391,19 +1418,7 @@ mod tests { #[tokio::test] async fn producing_a_gloas_slot_is_a_400_even_with_no_execution_client() { let (store, head_slot) = gloas_scheduled_store(); - let app = routes() - .with_state(store) - .layer(Extension(None::)) - .layer(Extension(SharedSyncCommitteePool::default())) - .layer(Extension(FeeRecipients::default())) - .layer(Extension(OwnVersion(Arc::new( - ethlambda_engine::types::ClientVersionV1 { - code: "LA".to_string(), - name: "ethlambda".to_string(), - version: "ethlambda/test".to_string(), - commit: "0x3c4d7e8f".to_string(), - }, - )))); + let app = produce_app(store, None); let uri = format!( "/eth/v3/validator/blocks/{}?randao_reveal=0x{}", head_slot + 1, @@ -1807,8 +1822,12 @@ mod tests { }); } + /// The production wiring: the router and the builder read one sync + /// status. fn app(&self, engine: Option) -> Router { - produce_app(self.store.clone(), engine).layer(Extension(self.builder.clone())) + let sync_status = self.builder.sync_status.clone(); + produce_app_with_sync(self.store.clone(), engine, sync_status) + .layer(Extension(self.builder.clone())) } fn uri(&self, query: &str) -> String { @@ -2032,12 +2051,22 @@ mod tests { answer.assert(&world, false, "42", 0); } + /// Refused before the head is read, so neither the execution client nor + /// the builder is asked. The breaker's own syncing trip is covered in + /// `breaker`. #[tokio::test] - async fn a_syncing_node_keeps_the_builder_out() { + async fn a_syncing_node_answers_503_and_asks_no_builder() { let world = builder_world(true, SyncStatus::Syncing).await; world.offer(ETH, |_| {}); let answer = produce(&world, "0x2a", false, "").await; - answer.assert(&world, false, "42", 0); + assert_eq!( + answer.status, + StatusCode::SERVICE_UNAVAILABLE, + "{}", + answer.json + ); + assert_eq!(answer.json["message"], "the node is syncing"); + assert!(world.relay.header_requests().is_empty()); } #[tokio::test] diff --git a/docs/rpc.md b/docs/rpc.md index 1712a24a4..dc50750d2 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -453,8 +453,10 @@ the chain actor writes, so no request waits on the actor. root comes from running the block through `process_block`. The answer is fulu `BlockContents`: the block, the bundle's cell proofs and its blobs, with `Eth-Execution-Payload-Blinded: false`. It is a - **`503`** without a configured execution client, or when the bundle is - malformed (its commitments, proofs and blobs disagree in count). A slot the + **`503`** while the node is syncing (checked before the head is read, as + for the duties endpoints), without a configured execution client, or when + the bundle is malformed (its commitments, proofs and blobs disagree in + count). The head state is advanced to the slot on a blocking thread. A slot the schedule does not place at fulu (gloas included) is a `400`, checked before the execution client; `produceBlockV3` is the fulu endpoint, and a gloas slot goes to `produceBlockV4`. So is a slot that has not started yet: one starting @@ -609,7 +611,7 @@ entries are decoded and not consulted. | `200` JSON | `{version: "gloas", consensus_block_value, execution_payload_value, execution_payload_included, data}`, where `data` is the same container as the SSZ body | | Response headers | `Eth-Consensus-Version: gloas`, `Eth-Execution-Payload-Included` (`true` or `false`; always `false` for a bid-won block), `Eth-Execution-Payload-Value` (wei, decimal; a bid's value times `10^9` when it won), `Eth-Consensus-Block-Value` (always `0`: the comparison is of execution payload values only). No `Eth-Builder-Url`: that is for builder-API bids | | `400` | A slot the schedule does not place at gloas or that has not started yet (more than 500 ms ahead of this node's clock, as for `produceBlockV3`), a missing or malformed query, a wrong `Eth-Consensus-Version`, a bad body, a slot not after the head block, or a `randao_reveal` that does not verify against the slot's proposer. The slot check precedes the execution-client check, so it is a `400` on any node | -| `503` | No execution client and no viable pooled bid for the slot, the execution client did not start or return a build (and no bid took its place), the node is building on a FULL parent whose envelope it does not hold, or the winning bid failed to build and there is no local payload to fall back on | +| `503` | The node is syncing (checked before the head is read), no execution client and no viable pooled bid for the slot, the execution client did not start or return a build (and no bid took its place), the node is building on a FULL parent whose envelope it does not hold, or the winning bid failed to build and there is no local payload to fall back on | The node advances the head state to the slot, and decides which parent payload to build on with `should_build_on_full` over the payload status fork choice