From 9a771ce3558674f5c178e85a5c1fb01e21393b15 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 18:48:53 +0800 Subject: [PATCH 01/12] Add execution proof observability --- beacon_node/beacon_chain/src/beacon_chain.rs | 47 ++++++++-- .../gossip_verified_execution_proof.rs | 62 +++++++++---- .../src/execution_proof_verification/mod.rs | 55 ++++++++++++ beacon_node/beacon_chain/src/metrics.rs | 90 ++++++++++++++++++- .../src/pending_payload_cache/mod.rs | 35 ++++++++ .../http_api/src/beacon/execution_proofs.rs | 17 ++++ .../gossip_methods.rs | 19 ++++ 7 files changed, 302 insertions(+), 23 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 8a2e0218ee8..165b7a600ca 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4247,12 +4247,47 @@ impl BeaconChain { verified_proof: GossipVerifiedExecutionProof, ) -> Result { let GossipVerifiedExecutionProof { proof, block_slot } = verified_proof; - let availability = self - .pending_payload_cache - .put_execution_proof(proof) - .map_err(BlockError::from)?; - self.process_payload_envelope_availability(block_slot, availability, || Ok(())) - .await + let block_root = proof.beacon_block_root(); + let proof_type = proof.proof_type(); + let availability = match self.pending_payload_cache.put_execution_proof(proof) { + Ok(availability) => availability, + Err(error) => { + metrics::inc_counter_vec( + &metrics::EXECUTION_PROOF_AVAILABILITY_TOTAL, + &["cache_error"], + ); + debug!( + %block_root, + %proof_type, + ?error, + "Could not cache verified execution proof" + ); + return Err(BlockError::from(error)); + } + }; + let result = self + .process_payload_envelope_availability(block_slot, availability, || Ok(())) + .await; + let outcome = match &result { + Ok(AvailabilityProcessingStatus::Imported(..)) => "imported", + Ok(AvailabilityProcessingStatus::MissingComponents(..)) => "missing_components", + Err(_) => "import_error", + }; + metrics::inc_counter_vec(&metrics::EXECUTION_PROOF_AVAILABILITY_TOTAL, &[outcome]); + debug!( + %block_root, + %proof_type, + outcome, + cached_execution_proofs = self + .pending_payload_cache + .get_execution_proofs(&block_root) + .len(), + required_execution_proofs = self + .pending_payload_cache + .required_execution_proofs(), + "Processed execution proof availability" + ); + result } fn check_data_column_sidecar_header_signature_and_slashability<'a>( diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs index 611e9c6ec51..81ed1e8bf11 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs @@ -4,6 +4,7 @@ use crate::canonical_head::CanonicalHead; use crate::execution_proof_verification::observed_execution_proofs::{ ObservedExecutionProofs, ProofObservation, }; +use crate::metrics; use crate::pending_payload_cache::PendingPayloadCache; use crate::shuffling_cache::{ShufflingCache, with_cached_shuffling}; use crate::validator_pubkey_cache::ValidatorPubkeyCache; @@ -15,6 +16,7 @@ use ssz_types::ProgressiveVariableList; use state_processing::builder_deposits_cache::OnboardBuildersCache; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; +use std::time::Instant; use tree_hash::TreeHash; use types::execution::{ExecutionProof, ExecutionProofEnvelope, SignedExecutionProofEnvelope}; use types::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot, Slot}; @@ -169,10 +171,22 @@ impl GossipVerifiedExecutionProof { // [REJECT] The proof verifies via the proof engine. let proof_engine = ctx.proof_engine.as_ref().ok_or(Error::ProofEngineMissing)?; - match proof_engine - .verify_execution_proof(&execution_proof) - .map_err(Error::ProofEngine)? - { + let proof_engine_started = Instant::now(); + let proof_engine_result = proof_engine.verify_execution_proof(&execution_proof); + let proof_engine_outcome = match &proof_engine_result { + Ok(ProofVerificationOutcome::Valid) => "valid", + Ok(ProofVerificationOutcome::Invalid) => "invalid", + Err(_) => "error", + }; + metrics::observe_timer_vec( + &metrics::EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS, + &[ + metrics::execution_proof_type_label(proof_type), + proof_engine_outcome, + ], + proof_engine_started.elapsed(), + ); + match proof_engine_result.map_err(Error::ProofEngine)? { ProofVerificationOutcome::Invalid => return Err(Error::InvalidProof), ProofVerificationOutcome::Valid => {} } @@ -260,19 +274,35 @@ impl BeaconChain { self: &Arc, proof: Arc, ) -> Result { + let proof_type = proof.proof_type(); let chain = self.clone(); - self.task_executor - .clone() - .spawn_blocking_handle( - move || { - let ctx = chain.execution_proof_gossip_verification_context(); - GossipVerifiedExecutionProof::new(proof, &ctx) - }, - "gossip_execution_proof_verification_handle", - ) - .ok_or(BeaconChainError::RuntimeShutdown)? - .await - .map_err(BeaconChainError::TokioJoin)? + let verification_handle = self.task_executor.clone().spawn_blocking_handle( + move || { + let ctx = chain.execution_proof_gossip_verification_context(); + GossipVerifiedExecutionProof::new(proof, &ctx) + }, + "gossip_execution_proof_verification_handle", + ); + let result = match verification_handle { + Some(handle) => match handle.await { + Ok(result) => result, + Err(error) => Err(BeaconChainError::TokioJoin(error).into()), + }, + None => Err(BeaconChainError::RuntimeShutdown.into()), + }; + let (outcome, reason) = result + .as_ref() + .map(|_| ("accepted", "valid")) + .unwrap_or_else(|error| error.metric_labels()); + metrics::inc_counter_vec( + &metrics::EXECUTION_PROOF_VERIFICATION_TOTAL, + &[ + metrics::execution_proof_type_label(proof_type), + outcome, + reason, + ], + ); + result } } diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs b/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs index f1b05c06f0d..f183fad2472 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs @@ -56,6 +56,28 @@ pub enum Error { BeaconChainError(Box), } +impl Error { + /// Stable, bounded labels describing how gossip verification classified this error. + pub const fn metric_labels(&self) -> (&'static str, &'static str) { + match self { + Self::ProofAlreadySeen => ("ignored", "proof_already_seen"), + Self::ValidProofAlreadyKnown => ("ignored", "valid_proof_already_known"), + Self::DuplicateFromValidator { .. } => ("ignored", "duplicate_from_validator"), + Self::UnknownBlockRoot { .. } => ("ignored", "unknown_block_root"), + Self::PastFinalizedSlot { .. } => ("ignored", "past_finalized_slot"), + Self::PayloadUnavailable { .. } => ("ignored", "payload_unavailable"), + Self::EmptyProofData => ("rejected", "empty_proof_data"), + Self::UnknownValidatorIndex(_) => ("rejected", "unknown_validator_index"), + Self::ValidatorNotActive { .. } => ("rejected", "validator_not_active"), + Self::InvalidSignature => ("rejected", "invalid_signature"), + Self::InvalidProof => ("rejected", "invalid_proof"), + Self::ProofEngineMissing => ("error", "proof_engine_missing"), + Self::ProofEngine(_) => ("error", "proof_engine"), + Self::BeaconChainError(_) => ("error", "beacon_chain"), + } + } +} + impl From for Error { fn from(e: BeaconChainError) -> Self { Error::BeaconChainError(Box::new(e)) @@ -75,3 +97,36 @@ impl From for Error { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn metric_labels_follow_gossip_classification() { + assert_eq!( + Error::ProofAlreadySeen.metric_labels(), + ("ignored", "proof_already_seen") + ); + assert_eq!( + Error::PayloadUnavailable { + beacon_block_root: Hash256::default(), + } + .metric_labels(), + ("ignored", "payload_unavailable") + ); + assert_eq!( + Error::InvalidSignature.metric_labels(), + ("rejected", "invalid_signature") + ); + assert_eq!( + Error::ProofEngine(ProofEngineError::ProofVerifierError("failed".to_string())) + .metric_labels(), + ("error", "proof_engine") + ); + assert_eq!( + Error::BeaconChainError(Box::new(BeaconChainError::RuntimeShutdown)).metric_labels(), + ("error", "beacon_chain") + ); + } +} diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index d4880549848..7c9a2ac7385 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -5,7 +5,7 @@ use bls::FixedBytesExtended; pub use metrics::*; use slot_clock::SlotClock; use std::sync::LazyLock; -use types::{BeaconState, Epoch, EthSpec, Hash256, Slot}; +use types::{BeaconState, Epoch, EthSpec, Hash256, Slot, execution::ProofType}; // Attestation simulator metrics pub const VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_HEAD_ATTESTER_HIT_TOTAL: &str = @@ -2110,6 +2110,76 @@ pub static PENDING_PAYLOAD_CACHE_SIZE: LazyLock> = LazyLock::ne "Number of entries in the pending payload availability cache.", ) }); +pub static PENDING_PAYLOAD_CACHE_EXECUTION_PROOFS: LazyLock> = + LazyLock::new(|| { + try_create_int_gauge( + "pending_payload_cache_execution_proofs", + "Number of distinct execution proofs stored in the pending payload cache.", + ) + }); +pub static PENDING_PAYLOAD_CACHE_PAYLOADS_AWAITING_EXECUTION_PROOFS: LazyLock> = + LazyLock::new(|| { + try_create_int_gauge( + "pending_payload_cache_payloads_awaiting_execution_proofs", + "Number of executed payload envelopes waiting for more distinct execution proofs.", + ) + }); +pub static PENDING_PAYLOAD_CACHE_REQUIRED_EXECUTION_PROOFS: LazyLock> = + LazyLock::new(|| { + try_create_int_gauge( + "pending_payload_cache_required_execution_proofs", + "Configured number of distinct execution proofs required before payload import.", + ) + }); + +/* + * Execution proof metrics + */ +pub static EXECUTION_PROOF_RECEIVED_TOTAL: LazyLock> = LazyLock::new(|| { + try_create_int_counter_vec( + "beacon_execution_proof_received_total", + "Count of execution proof envelopes received by source and proof type.", + &["source", "proof_type"], + ) +}); +pub static EXECUTION_PROOF_VERIFICATION_TOTAL: LazyLock> = + LazyLock::new(|| { + try_create_int_counter_vec( + "beacon_execution_proof_verification_total", + "Count of execution proof verification outcomes by proof type, outcome, and reason.", + &["proof_type", "outcome", "reason"], + ) + }); +pub static EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS: LazyLock> = + LazyLock::new(|| { + try_create_histogram_vec_with_buckets( + "beacon_execution_proof_engine_verification_seconds", + "Time spent verifying execution proofs in the proof engine by proof type and outcome.", + decimal_buckets(-3, 1), + &["proof_type", "outcome"], + ) + }); +pub static EXECUTION_PROOF_AVAILABILITY_TOTAL: LazyLock> = + LazyLock::new(|| { + try_create_int_counter_vec( + "beacon_execution_proof_availability_total", + "Count of payload availability outcomes triggered by accepted execution proofs.", + &["outcome"], + ) + }); + +/// Stable, bounded Prometheus label for an assigned EIP-8025 proof type. +pub const fn execution_proof_type_label(proof_type: ProofType) -> &'static str { + match proof_type { + ProofType::EthrexOpenVM => "ethrex_openvm", + ProofType::EthrexSP1 => "ethrex_sp1", + ProofType::EthrexZisk => "ethrex_zisk", + ProofType::RethOpenVM => "reth_openvm", + ProofType::RethSP1 => "reth_sp1", + ProofType::RethZisk => "reth_zisk", + ProofType::ZesuZisk => "zesu_zisk", + } +} pub static DATA_AVAILABILITY_RECONSTRUCTION_TIME: LazyLock> = LazyLock::new(|| { try_create_histogram( @@ -2221,6 +2291,24 @@ pub fn scrape_for_metrics(beacon_chain: &BeaconChain) { &PENDING_PAYLOAD_CACHE_SIZE, beacon_chain.pending_payload_cache.cache_size(), ); + set_gauge_by_usize( + &PENDING_PAYLOAD_CACHE_EXECUTION_PROOFS, + beacon_chain + .pending_payload_cache + .cached_execution_proof_count(), + ); + set_gauge_by_usize( + &PENDING_PAYLOAD_CACHE_PAYLOADS_AWAITING_EXECUTION_PROOFS, + beacon_chain + .pending_payload_cache + .payloads_awaiting_execution_proofs(), + ); + set_gauge_by_usize( + &PENDING_PAYLOAD_CACHE_REQUIRED_EXECUTION_PROOFS, + beacon_chain + .pending_payload_cache + .required_execution_proofs(), + ); if let Some((size, num_lookups)) = beacon_chain.pre_finalization_block_cache.metrics() { set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_CACHE_SIZE, size); diff --git a/beacon_node/beacon_chain/src/pending_payload_cache/mod.rs b/beacon_node/beacon_chain/src/pending_payload_cache/mod.rs index 0401c93d263..878e095a162 100644 --- a/beacon_node/beacon_chain/src/pending_payload_cache/mod.rs +++ b/beacon_node/beacon_chain/src/pending_payload_cache/mod.rs @@ -579,6 +579,31 @@ impl PendingPayloadCache { self.availability_cache.read().len() } + /// Number of distinct execution proofs held across all pending payloads. + pub fn cached_execution_proof_count(&self) -> usize { + self.availability_cache + .read() + .iter() + .map(|(_, components)| components.execution_proofs.len()) + .sum() + } + + /// Number of executed payload envelopes that remain gated on execution proofs. + pub fn payloads_awaiting_execution_proofs(&self) -> usize { + if self.required_execution_proofs == 0 { + return 0; + } + + self.availability_cache + .read() + .iter() + .filter(|(_, components)| { + components.envelope.is_some() + && components.execution_proofs.len() < self.required_execution_proofs + }) + .count() + } + // ── Internal helpers ── fn check_availability( @@ -1038,7 +1063,10 @@ mod data_availability_checker_tests { #[tokio::test] async fn execution_proof_gates_availability() { let s = setup_gated(NodeCustodyType::Fullnode); + assert_eq!(s.cache.cached_execution_proof_count(), 0); + assert_eq!(s.cache.payloads_awaiting_execution_proofs(), 0); s.put_envelope(); + assert_eq!(s.cache.payloads_awaiting_execution_proofs(), 1); assert_missing(s.put_columns(s.custody.clone())); let assigned = ProofType::all(); @@ -1047,6 +1075,8 @@ mod data_availability_checker_tests { for _ in 0..=REQUIRED_EXECUTION_PROOFS { assert_missing(s.put_proof(assigned[0])); } + assert_eq!(s.cache.cached_execution_proof_count(), 1); + assert_eq!(s.cache.payloads_awaiting_execution_proofs(), 1); // Distinct provers up to the requirement flip it to available. let mut availability = None; @@ -1055,6 +1085,11 @@ mod data_availability_checker_tests { } let envelope = assert_available(availability.expect("gate needs two provers or more")); assert_eq!(envelope.block_root, s.block_root); + assert_eq!( + s.cache.cached_execution_proof_count(), + REQUIRED_EXECUTION_PROOFS + ); + assert_eq!(s.cache.payloads_awaiting_execution_proofs(), 0); assert_missing(s.put_proof(assigned[REQUIRED_EXECUTION_PROOFS])); } diff --git a/beacon_node/http_api/src/beacon/execution_proofs.rs b/beacon_node/http_api/src/beacon/execution_proofs.rs index 25464bd2003..5a47c7eb737 100644 --- a/beacon_node/http_api/src/beacon/execution_proofs.rs +++ b/beacon_node/http_api/src/beacon/execution_proofs.rs @@ -16,6 +16,7 @@ use crate::utils::{ }; use crate::version::add_ssz_content_type_header; use beacon_chain::execution_proof_verification::Error as ProofError; +use beacon_chain::metrics as beacon_chain_metrics; use beacon_chain::{AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes}; use bytes::Bytes; use eth2::types::{self as api_types, Failure}; @@ -224,6 +225,22 @@ async fn publish_execution_proof( let beacon_block_root = proof.beacon_block_root(); let proof_type = proof.proof_type(); let validator_index = proof.validator_index; + let proof_bytes = proof.message.proof_data.len(); + + beacon_chain_metrics::inc_counter_vec( + &beacon_chain_metrics::EXECUTION_PROOF_RECEIVED_TOTAL, + &[ + "http_api", + beacon_chain_metrics::execution_proof_type_label(proof_type), + ], + ); + debug!( + %beacon_block_root, + %proof_type, + validator_index, + proof_bytes, + "Received execution proof from HTTP API" + ); let verified = match chain.verify_execution_proof_for_gossip(proof.clone()).await { Ok(verified) => verified, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index e27043614e1..e20f73907a6 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -12,6 +12,7 @@ use beacon_chain::data_column_verification::{ }; use beacon_chain::execution_proof_verification::Error as ExecutionProofError; use beacon_chain::fetch_blobs::PartialHeaderOrBid; +use beacon_chain::metrics as beacon_chain_metrics; use beacon_chain::partial_data_column_assembler::UpdatedPartials; use beacon_chain::payload_bid_verification::PayloadBidError; use beacon_chain::payload_envelope_verification::{ @@ -4167,6 +4168,24 @@ impl NetworkBeaconProcessor { ) { let beacon_block_root = execution_proof.beacon_block_root(); let proof_type = execution_proof.proof_type(); + let validator_index = execution_proof.validator_index; + let proof_bytes = execution_proof.message.proof_data.len(); + + beacon_chain_metrics::inc_counter_vec( + &beacon_chain_metrics::EXECUTION_PROOF_RECEIVED_TOTAL, + &[ + "gossip", + beacon_chain_metrics::execution_proof_type_label(proof_type), + ], + ); + debug!( + %peer_id, + %beacon_block_root, + %proof_type, + validator_index, + proof_bytes, + "Received execution proof from gossip" + ); match self .chain From 9877ff5754fd290736fea742da4bf14425725807 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 18:57:57 +0800 Subject: [PATCH 02/12] Refine proof engine metric labels --- .../gossip_verified_execution_proof.rs | 10 ++--- beacon_node/beacon_chain/src/metrics.rs | 4 +- beacon_node/proof_engine/src/lib.rs | 43 +++++++++++++++++++ 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs index 81ed1e8bf11..97afb08ab8c 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs @@ -173,16 +173,16 @@ impl GossipVerifiedExecutionProof { let proof_engine = ctx.proof_engine.as_ref().ok_or(Error::ProofEngineMissing)?; let proof_engine_started = Instant::now(); let proof_engine_result = proof_engine.verify_execution_proof(&execution_proof); - let proof_engine_outcome = match &proof_engine_result { - Ok(ProofVerificationOutcome::Valid) => "valid", - Ok(ProofVerificationOutcome::Invalid) => "invalid", - Err(_) => "error", + let (proof_engine_outcome, proof_engine_error_type) = match &proof_engine_result { + Ok(outcome) => (outcome.to_string(), "none"), + Err(error) => ("error".to_string(), error.kind()), }; metrics::observe_timer_vec( &metrics::EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS, &[ metrics::execution_proof_type_label(proof_type), - proof_engine_outcome, + &proof_engine_outcome, + proof_engine_error_type, ], proof_engine_started.elapsed(), ); diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 7c9a2ac7385..60f7bfe3f53 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -2154,9 +2154,9 @@ pub static EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS: LazyLock> = diff --git a/beacon_node/proof_engine/src/lib.rs b/beacon_node/proof_engine/src/lib.rs index 1adfcf0bac5..1eb3a6ea0e4 100644 --- a/beacon_node/proof_engine/src/lib.rs +++ b/beacon_node/proof_engine/src/lib.rs @@ -5,6 +5,7 @@ mod config; pub mod ere; pub mod test_utils; +use std::fmt; use std::sync::Arc; use types::execution::{ExecutionProof, ProofType}; @@ -19,6 +20,16 @@ pub enum ProofEngineError { UnconfiguredProofType(ProofType), } +impl ProofEngineError { + /// Stable, bounded label identifying the error variant. + pub const fn kind(&self) -> &'static str { + match self { + Self::ProofVerifierError(_) => "proof_verifier_error", + Self::UnconfiguredProofType(_) => "unconfigured_proof_type", + } + } +} + /// Outcome of proof verification. `Invalid` means the artifact does not verify; it says nothing /// about the validity of the payload it claims to prove. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -29,6 +40,15 @@ pub enum ProofVerificationOutcome { Invalid, } +impl fmt::Display for ProofVerificationOutcome { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Valid => "valid", + Self::Invalid => "invalid", + }) + } +} + /// Interface used by the beacon chain to verify reconstructed execution proofs. pub trait ProofEngineT: Send + Sync + 'static { /// Verify a reconstructed execution proof. @@ -66,3 +86,26 @@ impl ProofEngine { self.inner.verify_execution_proof(proof) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn proof_verification_outcome_display_is_stable() { + assert_eq!(ProofVerificationOutcome::Valid.to_string(), "valid"); + assert_eq!(ProofVerificationOutcome::Invalid.to_string(), "invalid"); + } + + #[test] + fn proof_engine_error_kinds_are_stable() { + assert_eq!( + ProofEngineError::ProofVerifierError("failed".to_string()).kind(), + "proof_verifier_error" + ); + assert_eq!( + ProofEngineError::UnconfiguredProofType(ProofType::RethSP1).kind(), + "unconfigured_proof_type" + ); + } +} From cead0a48f7c420f9f05a5735ec0320a7fa24ee6d Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 19:29:22 +0800 Subject: [PATCH 03/12] Simplify proof engine metric labels --- .../gossip_verified_execution_proof.rs | 12 +++++++----- beacon_node/proof_engine/src/lib.rs | 19 +++++++------------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs index 97afb08ab8c..8de5bc9822b 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs @@ -173,16 +173,18 @@ impl GossipVerifiedExecutionProof { let proof_engine = ctx.proof_engine.as_ref().ok_or(Error::ProofEngineMissing)?; let proof_engine_started = Instant::now(); let proof_engine_result = proof_engine.verify_execution_proof(&execution_proof); - let (proof_engine_outcome, proof_engine_error_type) = match &proof_engine_result { - Ok(outcome) => (outcome.to_string(), "none"), - Err(error) => ("error".to_string(), error.kind()), - }; + let proof_engine_outcome = proof_engine_result + .as_ref() + .map_or_else(|_| "error".to_string(), ToString::to_string); + let proof_engine_error_type = proof_engine_result + .as_ref() + .map_or_else(ToString::to_string, |_| "none".to_string()); metrics::observe_timer_vec( &metrics::EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS, &[ metrics::execution_proof_type_label(proof_type), &proof_engine_outcome, - proof_engine_error_type, + &proof_engine_error_type, ], proof_engine_started.elapsed(), ); diff --git a/beacon_node/proof_engine/src/lib.rs b/beacon_node/proof_engine/src/lib.rs index 1eb3a6ea0e4..cabc6096ca9 100644 --- a/beacon_node/proof_engine/src/lib.rs +++ b/beacon_node/proof_engine/src/lib.rs @@ -20,13 +20,12 @@ pub enum ProofEngineError { UnconfiguredProofType(ProofType), } -impl ProofEngineError { - /// Stable, bounded label identifying the error variant. - pub const fn kind(&self) -> &'static str { - match self { +impl fmt::Display for ProofEngineError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { Self::ProofVerifierError(_) => "proof_verifier_error", Self::UnconfiguredProofType(_) => "unconfigured_proof_type", - } + }) } } @@ -92,19 +91,15 @@ mod tests { use super::*; #[test] - fn proof_verification_outcome_display_is_stable() { + fn proof_verification_labels_are_stable() { assert_eq!(ProofVerificationOutcome::Valid.to_string(), "valid"); assert_eq!(ProofVerificationOutcome::Invalid.to_string(), "invalid"); - } - - #[test] - fn proof_engine_error_kinds_are_stable() { assert_eq!( - ProofEngineError::ProofVerifierError("failed".to_string()).kind(), + ProofEngineError::ProofVerifierError("failed".to_string()).to_string(), "proof_verifier_error" ); assert_eq!( - ProofEngineError::UnconfiguredProofType(ProofType::RethSP1).kind(), + ProofEngineError::UnconfiguredProofType(ProofType::RethSP1).to_string(), "unconfigured_proof_type" ); } From 9310e52480a799e23e11144079279a52e9b08eb1 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 19:34:27 +0800 Subject: [PATCH 04/12] Clarify proof engine metric labels --- .../gossip_verified_execution_proof.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs index 8de5bc9822b..45fa272c5cf 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs @@ -173,12 +173,10 @@ impl GossipVerifiedExecutionProof { let proof_engine = ctx.proof_engine.as_ref().ok_or(Error::ProofEngineMissing)?; let proof_engine_started = Instant::now(); let proof_engine_result = proof_engine.verify_execution_proof(&execution_proof); - let proof_engine_outcome = proof_engine_result - .as_ref() - .map_or_else(|_| "error".to_string(), ToString::to_string); - let proof_engine_error_type = proof_engine_result - .as_ref() - .map_or_else(ToString::to_string, |_| "none".to_string()); + let (proof_engine_outcome, proof_engine_error_type) = match &proof_engine_result { + Ok(outcome) => (outcome.to_string(), "none".to_string()), + Err(error) => ("error".to_string(), error.to_string()), + }; metrics::observe_timer_vec( &metrics::EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS, &[ From ca8b5137b1f724772b05d2b2c0bd1ddfadcd9189 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 19:55:46 +0800 Subject: [PATCH 05/12] Use stable proof engine error labels --- .../gossip_verified_execution_proof.rs | 8 ++-- .../src/execution_proof_verification/mod.rs | 7 +++- beacon_node/proof_engine/src/config.rs | 7 ++-- beacon_node/proof_engine/src/ere/bindings.rs | 32 +++++++++++----- beacon_node/proof_engine/src/ere/mod.rs | 22 +++++++---- beacon_node/proof_engine/src/lib.rs | 38 +++++++++++-------- 6 files changed, 72 insertions(+), 42 deletions(-) diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs index 45fa272c5cf..a93626cff7b 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs @@ -174,15 +174,15 @@ impl GossipVerifiedExecutionProof { let proof_engine_started = Instant::now(); let proof_engine_result = proof_engine.verify_execution_proof(&execution_proof); let (proof_engine_outcome, proof_engine_error_type) = match &proof_engine_result { - Ok(outcome) => (outcome.to_string(), "none".to_string()), - Err(error) => ("error".to_string(), error.to_string()), + Ok(outcome) => (outcome.as_str(), "none"), + Err(error) => ("error", error.as_str()), }; metrics::observe_timer_vec( &metrics::EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS, &[ metrics::execution_proof_type_label(proof_type), - &proof_engine_outcome, - &proof_engine_error_type, + proof_engine_outcome, + proof_engine_error_type, ], proof_engine_started.elapsed(), ); diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs b/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs index f183fad2472..311a36fd4fa 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs @@ -120,8 +120,11 @@ mod tests { ("rejected", "invalid_signature") ); assert_eq!( - Error::ProofEngine(ProofEngineError::ProofVerifierError("failed".to_string())) - .metric_labels(), + Error::ProofEngine(ProofEngineError::ProofVerifierError { + message: "failed".to_string(), + error_type: "internal", + }) + .metric_labels(), ("error", "proof_engine") ); assert_eq!( diff --git a/beacon_node/proof_engine/src/config.rs b/beacon_node/proof_engine/src/config.rs index fa2faa62645..b3fd57af361 100644 --- a/beacon_node/proof_engine/src/config.rs +++ b/beacon_node/proof_engine/src/config.rs @@ -123,9 +123,10 @@ impl ProofEngineConfig { #[cfg(not(feature = "ere-verifier"))] { - Err(ProofEngineError::ProofVerifierError( - "Lighthouse was built without `ere-verifier`".to_string(), - )) + Err(ProofEngineError::ProofVerifierError { + message: "Lighthouse was built without `ere-verifier`".to_string(), + error_type: "proof_verifier_error", + }) } } } diff --git a/beacon_node/proof_engine/src/ere/bindings.rs b/beacon_node/proof_engine/src/ere/bindings.rs index 4c876928d26..9401d979bd7 100644 --- a/beacon_node/proof_engine/src/ere/bindings.rs +++ b/beacon_node/proof_engine/src/ere/bindings.rs @@ -23,6 +23,18 @@ pub(super) enum EreVerifierError { } impl EreVerifierError { + pub(super) const fn as_str(&self) -> &'static str { + match self { + Self::NullPointer => "null_pointer", + Self::BadZkvmKind => "bad_zkvm_kind", + Self::DecodeProgramVk => "decode_program_vk", + Self::DecodeProof => "decode_proof", + Self::Verify => "verify", + Self::Internal => "internal", + Self::Unknown(_) => "unknown", + } + } + fn from_code(code: i32) -> Self { match code { 1 => Self::NullPointer, @@ -142,17 +154,19 @@ mod tests { #[test] fn decodes_ere_status_codes() { let statuses = [ - (1, EreVerifierError::NullPointer), - (2, EreVerifierError::BadZkvmKind), - (3, EreVerifierError::DecodeProgramVk), - (4, EreVerifierError::DecodeProof), - (5, EreVerifierError::Verify), - (6, EreVerifierError::Internal), - (99, EreVerifierError::Unknown(99)), + (1, EreVerifierError::NullPointer, "null_pointer"), + (2, EreVerifierError::BadZkvmKind, "bad_zkvm_kind"), + (3, EreVerifierError::DecodeProgramVk, "decode_program_vk"), + (4, EreVerifierError::DecodeProof, "decode_proof"), + (5, EreVerifierError::Verify, "verify"), + (6, EreVerifierError::Internal, "internal"), + (99, EreVerifierError::Unknown(99), "unknown"), ]; - for (code, expected) in statuses { - assert_eq!(EreVerifierError::from_code(code), expected); + for (code, expected, expected_label) in statuses { + let error = EreVerifierError::from_code(code); + assert_eq!(error, expected); + assert_eq!(error.as_str(), expected_label); } } } diff --git a/beacon_node/proof_engine/src/ere/mod.rs b/beacon_node/proof_engine/src/ere/mod.rs index 5b9354b7c8c..9bb97fbfacb 100644 --- a/beacon_node/proof_engine/src/ere/mod.rs +++ b/beacon_node/proof_engine/src/ere/mod.rs @@ -21,10 +21,13 @@ impl EreProofEngine { for config in config.execution_proofs() { let verifier = Verifier::new(config.proof_type.zkvm(), &config.program_vk).map_err(|error| { - ProofEngineError::ProofVerifierError(format!( - "failed to initialize ERE verifier for proof type {:?}: {error:?}", - config.proof_type - )) + ProofEngineError::ProofVerifierError { + message: format!( + "failed to initialize ERE verifier for proof type {:?}: {error:?}", + config.proof_type + ), + error_type: error.as_str(), + } })?; verifiers.insert(config.proof_type, verifier); } @@ -52,10 +55,13 @@ impl ProofEngineT for EreProofEngine { return Ok(ProofVerificationOutcome::Invalid); } Err(error) => { - return Err(ProofEngineError::ProofVerifierError(format!( - "ERE verifier failed for proof type {:?}: {error:?}", - proof.proof_type - ))); + return Err(ProofEngineError::ProofVerifierError { + message: format!( + "ERE verifier failed for proof type {:?}: {error:?}", + proof.proof_type + ), + error_type: error.as_str(), + }); } }; diff --git a/beacon_node/proof_engine/src/lib.rs b/beacon_node/proof_engine/src/lib.rs index cabc6096ca9..4b19347e0ec 100644 --- a/beacon_node/proof_engine/src/lib.rs +++ b/beacon_node/proof_engine/src/lib.rs @@ -5,7 +5,6 @@ mod config; pub mod ere; pub mod test_utils; -use std::fmt; use std::sync::Arc; use types::execution::{ExecutionProof, ProofType}; @@ -15,17 +14,20 @@ pub use config::{ExecutionProofConfig, ProofEngineConfig}; #[derive(Debug)] pub enum ProofEngineError { /// The configured proof verifier could not initialize or complete verification. - ProofVerifierError(String), + ProofVerifierError { + message: String, + error_type: &'static str, + }, /// No verifier is configured for the proof's EIP-8025 proof type. UnconfiguredProofType(ProofType), } -impl fmt::Display for ProofEngineError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match self { - Self::ProofVerifierError(_) => "proof_verifier_error", +impl ProofEngineError { + pub const fn as_str(&self) -> &'static str { + match self { + Self::ProofVerifierError { error_type, .. } => error_type, Self::UnconfiguredProofType(_) => "unconfigured_proof_type", - }) + } } } @@ -39,12 +41,12 @@ pub enum ProofVerificationOutcome { Invalid, } -impl fmt::Display for ProofVerificationOutcome { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match self { +impl ProofVerificationOutcome { + pub const fn as_str(&self) -> &'static str { + match self { Self::Valid => "valid", Self::Invalid => "invalid", - }) + } } } @@ -92,14 +94,18 @@ mod tests { #[test] fn proof_verification_labels_are_stable() { - assert_eq!(ProofVerificationOutcome::Valid.to_string(), "valid"); - assert_eq!(ProofVerificationOutcome::Invalid.to_string(), "invalid"); + assert_eq!(ProofVerificationOutcome::Valid.as_str(), "valid"); + assert_eq!(ProofVerificationOutcome::Invalid.as_str(), "invalid"); assert_eq!( - ProofEngineError::ProofVerifierError("failed".to_string()).to_string(), - "proof_verifier_error" + ProofEngineError::ProofVerifierError { + message: "failed".to_string(), + error_type: "internal", + } + .as_str(), + "internal" ); assert_eq!( - ProofEngineError::UnconfiguredProofType(ProofType::RethSP1).to_string(), + ProofEngineError::UnconfiguredProofType(ProofType::RethSP1).as_str(), "unconfigured_proof_type" ); } From f56bfb35dd2f2426ee7665f4c21c1f0ef985b943 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 21:54:44 +0800 Subject: [PATCH 06/12] Measure execution proof verification boundaries --- Cargo.lock | 1 + .../gossip_verified_execution_proof.rs | 28 ++++--------------- beacon_node/beacon_chain/src/metrics.rs | 22 ++++----------- .../http_api/src/beacon/execution_proofs.rs | 6 ++-- .../gossip_methods.rs | 10 ++++--- beacon_node/proof_engine/Cargo.toml | 1 + beacon_node/proof_engine/src/lib.rs | 16 ++++++++++- beacon_node/proof_engine/src/metrics.rs | 12 ++++++++ .../execution/execution_proof/proof_type.rs | 3 +- 9 files changed, 50 insertions(+), 49 deletions(-) create mode 100644 beacon_node/proof_engine/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index f37840497e9..17281298b46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7242,6 +7242,7 @@ dependencies = [ "execution_layer", "flate2", "hex", + "metrics", "reqwest", "serde", "serde_json", diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs index a93626cff7b..b5a8c05b431 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs @@ -16,7 +16,6 @@ use ssz_types::ProgressiveVariableList; use state_processing::builder_deposits_cache::OnboardBuildersCache; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; -use std::time::Instant; use tree_hash::TreeHash; use types::execution::{ExecutionProof, ExecutionProofEnvelope, SignedExecutionProofEnvelope}; use types::{ChainSpec, Domain, EthSpec, Hash256, SignedRoot, Slot}; @@ -171,22 +170,10 @@ impl GossipVerifiedExecutionProof { // [REJECT] The proof verifies via the proof engine. let proof_engine = ctx.proof_engine.as_ref().ok_or(Error::ProofEngineMissing)?; - let proof_engine_started = Instant::now(); - let proof_engine_result = proof_engine.verify_execution_proof(&execution_proof); - let (proof_engine_outcome, proof_engine_error_type) = match &proof_engine_result { - Ok(outcome) => (outcome.as_str(), "none"), - Err(error) => ("error", error.as_str()), - }; - metrics::observe_timer_vec( - &metrics::EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS, - &[ - metrics::execution_proof_type_label(proof_type), - proof_engine_outcome, - proof_engine_error_type, - ], - proof_engine_started.elapsed(), - ); - match proof_engine_result.map_err(Error::ProofEngine)? { + match proof_engine + .verify_execution_proof(&execution_proof) + .map_err(Error::ProofEngine)? + { ProofVerificationOutcome::Invalid => return Err(Error::InvalidProof), ProofVerificationOutcome::Valid => {} } @@ -275,6 +262,7 @@ impl BeaconChain { proof: Arc, ) -> Result { let proof_type = proof.proof_type(); + let proof_type_label: &'static str = proof_type.into(); let chain = self.clone(); let verification_handle = self.task_executor.clone().spawn_blocking_handle( move || { @@ -296,11 +284,7 @@ impl BeaconChain { .unwrap_or_else(|error| error.metric_labels()); metrics::inc_counter_vec( &metrics::EXECUTION_PROOF_VERIFICATION_TOTAL, - &[ - metrics::execution_proof_type_label(proof_type), - outcome, - reason, - ], + &[proof_type_label, outcome, reason], ); result } diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 60f7bfe3f53..12585d7ffb7 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -5,7 +5,7 @@ use bls::FixedBytesExtended; pub use metrics::*; use slot_clock::SlotClock; use std::sync::LazyLock; -use types::{BeaconState, Epoch, EthSpec, Hash256, Slot, execution::ProofType}; +use types::{BeaconState, Epoch, EthSpec, Hash256, Slot}; // Attestation simulator metrics pub const VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_HEAD_ATTESTER_HIT_TOTAL: &str = @@ -2150,13 +2150,13 @@ pub static EXECUTION_PROOF_VERIFICATION_TOTAL: LazyLock> = &["proof_type", "outcome", "reason"], ) }); -pub static EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS: LazyLock> = +pub static EXECUTION_PROOF_GOSSIP_PROCESSING_SECONDS: LazyLock> = LazyLock::new(|| { try_create_histogram_vec_with_buckets( - "beacon_execution_proof_engine_verification_seconds", - "Time spent verifying execution proofs in the proof engine by proof type, outcome, and error type.", + "beacon_execution_proof_gossip_processing_seconds", + "Time spent processing execution proofs received over gossip by proof type.", decimal_buckets(-3, 1), - &["proof_type", "outcome", "error_type"], + &["proof_type"], ) }); pub static EXECUTION_PROOF_AVAILABILITY_TOTAL: LazyLock> = @@ -2168,18 +2168,6 @@ pub static EXECUTION_PROOF_AVAILABILITY_TOTAL: LazyLock> = ) }); -/// Stable, bounded Prometheus label for an assigned EIP-8025 proof type. -pub const fn execution_proof_type_label(proof_type: ProofType) -> &'static str { - match proof_type { - ProofType::EthrexOpenVM => "ethrex_openvm", - ProofType::EthrexSP1 => "ethrex_sp1", - ProofType::EthrexZisk => "ethrex_zisk", - ProofType::RethOpenVM => "reth_openvm", - ProofType::RethSP1 => "reth_sp1", - ProofType::RethZisk => "reth_zisk", - ProofType::ZesuZisk => "zesu_zisk", - } -} pub static DATA_AVAILABILITY_RECONSTRUCTION_TIME: LazyLock> = LazyLock::new(|| { try_create_histogram( diff --git a/beacon_node/http_api/src/beacon/execution_proofs.rs b/beacon_node/http_api/src/beacon/execution_proofs.rs index 5a47c7eb737..dd20a377122 100644 --- a/beacon_node/http_api/src/beacon/execution_proofs.rs +++ b/beacon_node/http_api/src/beacon/execution_proofs.rs @@ -224,15 +224,13 @@ async fn publish_execution_proof( ) -> Result<(), ProofFailure> { let beacon_block_root = proof.beacon_block_root(); let proof_type = proof.proof_type(); + let proof_type_label: &'static str = proof_type.into(); let validator_index = proof.validator_index; let proof_bytes = proof.message.proof_data.len(); beacon_chain_metrics::inc_counter_vec( &beacon_chain_metrics::EXECUTION_PROOF_RECEIVED_TOTAL, - &[ - "http_api", - beacon_chain_metrics::execution_proof_type_label(proof_type), - ], + &["http_api", proof_type_label], ); debug!( %beacon_block_root, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index e20f73907a6..761d3c66671 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -4168,15 +4168,17 @@ impl NetworkBeaconProcessor { ) { let beacon_block_root = execution_proof.beacon_block_root(); let proof_type = execution_proof.proof_type(); + let proof_type_label: &'static str = proof_type.into(); + let _timer = beacon_chain_metrics::start_timer_vec( + &beacon_chain_metrics::EXECUTION_PROOF_GOSSIP_PROCESSING_SECONDS, + &[proof_type_label], + ); let validator_index = execution_proof.validator_index; let proof_bytes = execution_proof.message.proof_data.len(); beacon_chain_metrics::inc_counter_vec( &beacon_chain_metrics::EXECUTION_PROOF_RECEIVED_TOTAL, - &[ - "gossip", - beacon_chain_metrics::execution_proof_type_label(proof_type), - ], + &["gossip", proof_type_label], ); debug!( %peer_id, diff --git a/beacon_node/proof_engine/Cargo.toml b/beacon_node/proof_engine/Cargo.toml index 0d230ff143a..909dfb5b7d6 100644 --- a/beacon_node/proof_engine/Cargo.toml +++ b/beacon_node/proof_engine/Cargo.toml @@ -13,6 +13,7 @@ test-utils = [] ethereum_serde_utils = { workspace = true } ethereum_ssz = { workspace = true } hex = { workspace = true } +metrics = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } types = { workspace = true } diff --git a/beacon_node/proof_engine/src/lib.rs b/beacon_node/proof_engine/src/lib.rs index 4b19347e0ec..6ae56af45f0 100644 --- a/beacon_node/proof_engine/src/lib.rs +++ b/beacon_node/proof_engine/src/lib.rs @@ -3,9 +3,11 @@ mod config; #[cfg(feature = "ere-verifier")] pub mod ere; +mod metrics; pub mod test_utils; use std::sync::Arc; +use std::time::Instant; use types::execution::{ExecutionProof, ProofType}; pub use config::{ExecutionProofConfig, ProofEngineConfig}; @@ -84,7 +86,19 @@ impl ProofEngine { &self, proof: &ExecutionProof, ) -> Result { - self.inner.verify_execution_proof(proof) + let started = Instant::now(); + let result = self.inner.verify_execution_proof(proof); + let proof_type: &'static str = proof.proof_type.into(); + let (outcome, error_type) = match &result { + Ok(outcome) => (outcome.as_str(), "none"), + Err(error) => ("error", error.as_str()), + }; + metrics::observe_timer_vec( + &metrics::EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS, + &[proof_type, outcome, error_type], + started.elapsed(), + ); + result } } diff --git a/beacon_node/proof_engine/src/metrics.rs b/beacon_node/proof_engine/src/metrics.rs new file mode 100644 index 00000000000..768291b0f7b --- /dev/null +++ b/beacon_node/proof_engine/src/metrics.rs @@ -0,0 +1,12 @@ +pub use metrics::*; +use std::sync::LazyLock; + +pub static EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS: LazyLock> = + LazyLock::new(|| { + try_create_histogram_vec_with_buckets( + "beacon_execution_proof_engine_verification_seconds", + "Time spent verifying execution proofs in the proof engine by proof type, outcome, and error type.", + decimal_buckets(-3, 1), + &["proof_type", "outcome", "error_type"], + ) + }); diff --git a/consensus/types/src/execution/execution_proof/proof_type.rs b/consensus/types/src/execution/execution_proof/proof_type.rs index 1795d886e5c..3b28c13d8c6 100644 --- a/consensus/types/src/execution/execution_proof/proof_type.rs +++ b/consensus/types/src/execution/execution_proof/proof_type.rs @@ -1,7 +1,7 @@ use crate::Hash256; use serde::{Deserialize, Serialize}; use ssz::{Decode as SszDecode, DecodeError, Encode as SszEncode}; -use strum::{Display, FromRepr, VariantArray}; +use strum::{Display, FromRepr, IntoStaticStr, VariantArray}; use tree_hash::{PackedEncoding, TreeHash as TreeHashTrait, TreeHashType}; /// Proof system that verifies an execution proof. @@ -35,6 +35,7 @@ pub enum ZkvmKind { Hash, Display, FromRepr, + IntoStaticStr, VariantArray, Serialize, Deserialize, From 308a3a3352d28ed7ec7b60084193b857d0f7e5b5 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 21:54:50 +0800 Subject: [PATCH 07/12] Encode execution proof data as JSON hex --- consensus/types/src/execution/execution_proof/envelope.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/consensus/types/src/execution/execution_proof/envelope.rs b/consensus/types/src/execution/execution_proof/envelope.rs index 5ad9ec878b6..106a737280e 100644 --- a/consensus/types/src/execution/execution_proof/envelope.rs +++ b/consensus/types/src/execution/execution_proof/envelope.rs @@ -12,6 +12,7 @@ use tree_hash_derive::TreeHash; #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode, TreeHash)] #[context_deserialize(ForkName)] pub struct ExecutionProofEnvelope { + #[serde(with = "ssz_types::serde_utils::hex_var_list")] pub proof_data: ProofData, #[serde(with = "quoted_proof_type")] pub proof_type: ProofType, @@ -67,10 +68,11 @@ mod tests { } #[test] - fn signed_envelope_json_quotes_integers() { + fn signed_envelope_json_uses_beacon_api_encoding() { let envelope = signed_envelope(ProofType::RethSP1); let json = serde_json::to_value(&envelope).expect("serializes"); + assert_eq!(json["message"]["proof_data"], "0x01"); assert_eq!(json["message"]["proof_type"], "5"); assert_eq!(json["validator_index"], "7"); From 2c5800631fd956783738a223b06cb6c686b2ebc3 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 22:19:09 +0800 Subject: [PATCH 08/12] Simplify execution proof metric labels --- beacon_node/beacon_chain/src/beacon_chain.rs | 20 ++----- .../gossip_verified_execution_proof.rs | 56 ++++++++++++------- .../src/execution_proof_verification/mod.rs | 56 +++++++++---------- .../http_api/src/beacon/execution_proofs.rs | 3 +- .../gossip_methods.rs | 5 +- beacon_node/proof_engine/src/lib.rs | 3 +- 6 files changed, 70 insertions(+), 73 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 165b7a600ca..42f118eb6ac 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4249,22 +4249,10 @@ impl BeaconChain { let GossipVerifiedExecutionProof { proof, block_slot } = verified_proof; let block_root = proof.beacon_block_root(); let proof_type = proof.proof_type(); - let availability = match self.pending_payload_cache.put_execution_proof(proof) { - Ok(availability) => availability, - Err(error) => { - metrics::inc_counter_vec( - &metrics::EXECUTION_PROOF_AVAILABILITY_TOTAL, - &["cache_error"], - ); - debug!( - %block_root, - %proof_type, - ?error, - "Could not cache verified execution proof" - ); - return Err(BlockError::from(error)); - } - }; + let availability = self + .pending_payload_cache + .put_execution_proof(proof) + .map_err(BlockError::from)?; let result = self .process_payload_envelope_availability(block_slot, availability, || Ok(())) .await; diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs index b5a8c05b431..e0800c872d2 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs @@ -262,29 +262,47 @@ impl BeaconChain { proof: Arc, ) -> Result { let proof_type = proof.proof_type(); - let proof_type_label: &'static str = proof_type.into(); let chain = self.clone(); - let verification_handle = self.task_executor.clone().spawn_blocking_handle( - move || { - let ctx = chain.execution_proof_gossip_verification_context(); - GossipVerifiedExecutionProof::new(proof, &ctx) - }, - "gossip_execution_proof_verification_handle", - ); - let result = match verification_handle { - Some(handle) => match handle.await { - Ok(result) => result, - Err(error) => Err(BeaconChainError::TokioJoin(error).into()), - }, - None => Err(BeaconChainError::RuntimeShutdown.into()), + let result: Result<_, Error> = async { + self.task_executor + .clone() + .spawn_blocking_handle( + move || { + let ctx = chain.execution_proof_gossip_verification_context(); + GossipVerifiedExecutionProof::new(proof, &ctx) + }, + "gossip_execution_proof_verification_handle", + ) + .ok_or(BeaconChainError::RuntimeShutdown)? + .await + .map_err(BeaconChainError::TokioJoin)? + } + .await; + let (outcome, reason) = match &result { + Ok(_) => ("accepted", "valid"), + Err(error) => { + let outcome = match error { + Error::ProofAlreadySeen + | Error::ValidProofAlreadyKnown + | Error::DuplicateFromValidator { .. } + | Error::UnknownBlockRoot { .. } + | Error::PastFinalizedSlot { .. } + | Error::PayloadUnavailable { .. } => "ignored", + Error::EmptyProofData + | Error::UnknownValidatorIndex(_) + | Error::ValidatorNotActive { .. } + | Error::InvalidSignature + | Error::InvalidProof => "rejected", + Error::ProofEngineMissing + | Error::ProofEngine(_) + | Error::BeaconChainError(_) => "error", + }; + (outcome, error.as_str()) + } }; - let (outcome, reason) = result - .as_ref() - .map(|_| ("accepted", "valid")) - .unwrap_or_else(|error| error.metric_labels()); metrics::inc_counter_vec( &metrics::EXECUTION_PROOF_VERIFICATION_TOTAL, - &[proof_type_label, outcome, reason], + &[proof_type.into(), outcome, reason], ); result } diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs b/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs index 311a36fd4fa..023ff641da1 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs @@ -57,23 +57,23 @@ pub enum Error { } impl Error { - /// Stable, bounded labels describing how gossip verification classified this error. - pub const fn metric_labels(&self) -> (&'static str, &'static str) { + /// A stable, bounded description of this gossip verification error. + pub const fn as_str(&self) -> &'static str { match self { - Self::ProofAlreadySeen => ("ignored", "proof_already_seen"), - Self::ValidProofAlreadyKnown => ("ignored", "valid_proof_already_known"), - Self::DuplicateFromValidator { .. } => ("ignored", "duplicate_from_validator"), - Self::UnknownBlockRoot { .. } => ("ignored", "unknown_block_root"), - Self::PastFinalizedSlot { .. } => ("ignored", "past_finalized_slot"), - Self::PayloadUnavailable { .. } => ("ignored", "payload_unavailable"), - Self::EmptyProofData => ("rejected", "empty_proof_data"), - Self::UnknownValidatorIndex(_) => ("rejected", "unknown_validator_index"), - Self::ValidatorNotActive { .. } => ("rejected", "validator_not_active"), - Self::InvalidSignature => ("rejected", "invalid_signature"), - Self::InvalidProof => ("rejected", "invalid_proof"), - Self::ProofEngineMissing => ("error", "proof_engine_missing"), - Self::ProofEngine(_) => ("error", "proof_engine"), - Self::BeaconChainError(_) => ("error", "beacon_chain"), + Self::ProofAlreadySeen => "proof_already_seen", + Self::ValidProofAlreadyKnown => "valid_proof_already_known", + Self::DuplicateFromValidator { .. } => "duplicate_from_validator", + Self::UnknownBlockRoot { .. } => "unknown_block_root", + Self::PastFinalizedSlot { .. } => "past_finalized_slot", + Self::PayloadUnavailable { .. } => "payload_unavailable", + Self::EmptyProofData => "empty_proof_data", + Self::UnknownValidatorIndex(_) => "unknown_validator_index", + Self::ValidatorNotActive { .. } => "validator_not_active", + Self::InvalidSignature => "invalid_signature", + Self::InvalidProof => "invalid_proof", + Self::ProofEngineMissing => "proof_engine_missing", + Self::ProofEngine(_) => "proof_engine", + Self::BeaconChainError(_) => "beacon_chain", } } } @@ -103,33 +103,27 @@ mod tests { use super::*; #[test] - fn metric_labels_follow_gossip_classification() { - assert_eq!( - Error::ProofAlreadySeen.metric_labels(), - ("ignored", "proof_already_seen") - ); + fn error_labels_are_stable() { + assert_eq!(Error::ProofAlreadySeen.as_str(), "proof_already_seen"); assert_eq!( Error::PayloadUnavailable { beacon_block_root: Hash256::default(), } - .metric_labels(), - ("ignored", "payload_unavailable") - ); - assert_eq!( - Error::InvalidSignature.metric_labels(), - ("rejected", "invalid_signature") + .as_str(), + "payload_unavailable" ); + assert_eq!(Error::InvalidSignature.as_str(), "invalid_signature"); assert_eq!( Error::ProofEngine(ProofEngineError::ProofVerifierError { message: "failed".to_string(), error_type: "internal", }) - .metric_labels(), - ("error", "proof_engine") + .as_str(), + "proof_engine" ); assert_eq!( - Error::BeaconChainError(Box::new(BeaconChainError::RuntimeShutdown)).metric_labels(), - ("error", "beacon_chain") + Error::BeaconChainError(Box::new(BeaconChainError::RuntimeShutdown)).as_str(), + "beacon_chain" ); } } diff --git a/beacon_node/http_api/src/beacon/execution_proofs.rs b/beacon_node/http_api/src/beacon/execution_proofs.rs index dd20a377122..c2f0a19c47a 100644 --- a/beacon_node/http_api/src/beacon/execution_proofs.rs +++ b/beacon_node/http_api/src/beacon/execution_proofs.rs @@ -224,13 +224,12 @@ async fn publish_execution_proof( ) -> Result<(), ProofFailure> { let beacon_block_root = proof.beacon_block_root(); let proof_type = proof.proof_type(); - let proof_type_label: &'static str = proof_type.into(); let validator_index = proof.validator_index; let proof_bytes = proof.message.proof_data.len(); beacon_chain_metrics::inc_counter_vec( &beacon_chain_metrics::EXECUTION_PROOF_RECEIVED_TOTAL, - &["http_api", proof_type_label], + &["http_api", proof_type.into()], ); debug!( %beacon_block_root, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 761d3c66671..aae09b9edb9 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -4168,17 +4168,16 @@ impl NetworkBeaconProcessor { ) { let beacon_block_root = execution_proof.beacon_block_root(); let proof_type = execution_proof.proof_type(); - let proof_type_label: &'static str = proof_type.into(); let _timer = beacon_chain_metrics::start_timer_vec( &beacon_chain_metrics::EXECUTION_PROOF_GOSSIP_PROCESSING_SECONDS, - &[proof_type_label], + &[proof_type.into()], ); let validator_index = execution_proof.validator_index; let proof_bytes = execution_proof.message.proof_data.len(); beacon_chain_metrics::inc_counter_vec( &beacon_chain_metrics::EXECUTION_PROOF_RECEIVED_TOTAL, - &["gossip", proof_type_label], + &["gossip", proof_type.into()], ); debug!( %peer_id, diff --git a/beacon_node/proof_engine/src/lib.rs b/beacon_node/proof_engine/src/lib.rs index 6ae56af45f0..1f2e6c3a861 100644 --- a/beacon_node/proof_engine/src/lib.rs +++ b/beacon_node/proof_engine/src/lib.rs @@ -88,14 +88,13 @@ impl ProofEngine { ) -> Result { let started = Instant::now(); let result = self.inner.verify_execution_proof(proof); - let proof_type: &'static str = proof.proof_type.into(); let (outcome, error_type) = match &result { Ok(outcome) => (outcome.as_str(), "none"), Err(error) => ("error", error.as_str()), }; metrics::observe_timer_vec( &metrics::EXECUTION_PROOF_ENGINE_VERIFICATION_SECONDS, - &[proof_type, outcome, error_type], + &[proof.proof_type.into(), outcome, error_type], started.elapsed(), ); result From f2fb513d078f24610e08664cc6a8c5ab1549ef54 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 22:23:38 +0800 Subject: [PATCH 09/12] Add execution proof error outcomes --- .../gossip_verified_execution_proof.rs | 20 +--------------- .../src/execution_proof_verification/mod.rs | 24 ++++++++++++++++++- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs index e0800c872d2..e4d7e1bab02 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs @@ -280,25 +280,7 @@ impl BeaconChain { .await; let (outcome, reason) = match &result { Ok(_) => ("accepted", "valid"), - Err(error) => { - let outcome = match error { - Error::ProofAlreadySeen - | Error::ValidProofAlreadyKnown - | Error::DuplicateFromValidator { .. } - | Error::UnknownBlockRoot { .. } - | Error::PastFinalizedSlot { .. } - | Error::PayloadUnavailable { .. } => "ignored", - Error::EmptyProofData - | Error::UnknownValidatorIndex(_) - | Error::ValidatorNotActive { .. } - | Error::InvalidSignature - | Error::InvalidProof => "rejected", - Error::ProofEngineMissing - | Error::ProofEngine(_) - | Error::BeaconChainError(_) => "error", - }; - (outcome, error.as_str()) - } + Err(error) => (error.outcome(), error.as_str()), }; metrics::inc_counter_vec( &metrics::EXECUTION_PROOF_VERIFICATION_TOTAL, diff --git a/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs b/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs index 023ff641da1..3747aeefd19 100644 --- a/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs +++ b/beacon_node/beacon_chain/src/execution_proof_verification/mod.rs @@ -57,6 +57,24 @@ pub enum Error { } impl Error { + /// A stable classification of this gossip verification error. + pub const fn outcome(&self) -> &'static str { + match self { + Self::ProofAlreadySeen + | Self::ValidProofAlreadyKnown + | Self::DuplicateFromValidator { .. } + | Self::UnknownBlockRoot { .. } + | Self::PastFinalizedSlot { .. } + | Self::PayloadUnavailable { .. } => "ignored", + Self::EmptyProofData + | Self::UnknownValidatorIndex(_) + | Self::ValidatorNotActive { .. } + | Self::InvalidSignature + | Self::InvalidProof => "rejected", + Self::ProofEngineMissing | Self::ProofEngine(_) | Self::BeaconChainError(_) => "error", + } + } + /// A stable, bounded description of this gossip verification error. pub const fn as_str(&self) -> &'static str { match self { @@ -103,7 +121,11 @@ mod tests { use super::*; #[test] - fn error_labels_are_stable() { + fn error_metric_values_are_stable() { + assert_eq!(Error::ProofAlreadySeen.outcome(), "ignored"); + assert_eq!(Error::InvalidSignature.outcome(), "rejected"); + assert_eq!(Error::ProofEngineMissing.outcome(), "error"); + assert_eq!(Error::ProofAlreadySeen.as_str(), "proof_already_seen"); assert_eq!( Error::PayloadUnavailable { From 83e8c5ce1ff9a5e1d3735d5de838ed5f76afb407 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 22:27:45 +0800 Subject: [PATCH 10/12] Remove execution proof availability telemetry --- beacon_node/beacon_chain/src/beacon_chain.rs | 27 ++------------------ beacon_node/beacon_chain/src/metrics.rs | 9 ------- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 42f118eb6ac..8a2e0218ee8 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4247,35 +4247,12 @@ impl BeaconChain { verified_proof: GossipVerifiedExecutionProof, ) -> Result { let GossipVerifiedExecutionProof { proof, block_slot } = verified_proof; - let block_root = proof.beacon_block_root(); - let proof_type = proof.proof_type(); let availability = self .pending_payload_cache .put_execution_proof(proof) .map_err(BlockError::from)?; - let result = self - .process_payload_envelope_availability(block_slot, availability, || Ok(())) - .await; - let outcome = match &result { - Ok(AvailabilityProcessingStatus::Imported(..)) => "imported", - Ok(AvailabilityProcessingStatus::MissingComponents(..)) => "missing_components", - Err(_) => "import_error", - }; - metrics::inc_counter_vec(&metrics::EXECUTION_PROOF_AVAILABILITY_TOTAL, &[outcome]); - debug!( - %block_root, - %proof_type, - outcome, - cached_execution_proofs = self - .pending_payload_cache - .get_execution_proofs(&block_root) - .len(), - required_execution_proofs = self - .pending_payload_cache - .required_execution_proofs(), - "Processed execution proof availability" - ); - result + self.process_payload_envelope_availability(block_slot, availability, || Ok(())) + .await } fn check_data_column_sidecar_header_signature_and_slashability<'a>( diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 12585d7ffb7..23be2d74638 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -2159,15 +2159,6 @@ pub static EXECUTION_PROOF_GOSSIP_PROCESSING_SECONDS: LazyLock> = - LazyLock::new(|| { - try_create_int_counter_vec( - "beacon_execution_proof_availability_total", - "Count of payload availability outcomes triggered by accepted execution proofs.", - &["outcome"], - ) - }); - pub static DATA_AVAILABILITY_RECONSTRUCTION_TIME: LazyLock> = LazyLock::new(|| { try_create_histogram( From b0e0af286fa747491c444c8b5cd370c12eeb1ee3 Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 23:08:41 +0800 Subject: [PATCH 11/12] Fix execution proof network gossip race --- .../src/proving_execution_layer.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testing/execution_proof_network_tests/src/proving_execution_layer.rs b/testing/execution_proof_network_tests/src/proving_execution_layer.rs index b9fdbc6ed9e..45c5057d255 100644 --- a/testing/execution_proof_network_tests/src/proving_execution_layer.rs +++ b/testing/execution_proof_network_tests/src/proving_execution_layer.rs @@ -48,6 +48,9 @@ pub struct ProvenPayload { /// How long the prover waits for the beacon node to expose the block carrying a payload. const BLOCK_LOOKUP_TIMEOUT: Duration = Duration::from_secs(10); const BLOCK_LOOKUP_INTERVAL: Duration = Duration::from_millis(250); +/// Approximate mock proof-generation time. Without a delay, proof gossip can reach peers before +/// the block it references and be rejected as unknown. +const PROOF_GENERATION_DELAY: Duration = Duration::from_millis(500); struct Inner { upstream: String, @@ -202,6 +205,7 @@ impl Inner { .ok_or("no beacon node attached to the proving execution layer yet")?; let (block_root, slot) = self.find_block(&target, payload).await?; + tokio::time::sleep(PROOF_GENERATION_DELAY).await; let genesis_validators_root = target .get_beacon_genesis() .await From 90ee471b7b76eb12d36a7fb5bc981733a541568b Mon Sep 17 00:00:00 2001 From: frisitano Date: Mon, 21 Sep 2026 23:57:06 +0800 Subject: [PATCH 12/12] Stabilize execution proof network startup --- testing/execution_proof_network_tests/src/network.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/execution_proof_network_tests/src/network.rs b/testing/execution_proof_network_tests/src/network.rs index 85379b181d4..d0b2d34a418 100644 --- a/testing/execution_proof_network_tests/src/network.rs +++ b/testing/execution_proof_network_tests/src/network.rs @@ -166,7 +166,7 @@ impl ProofNetworkConfig { nodes, validator_count: 16, slot_duration_ms: 2000, - genesis_delay_secs: 30, + genesis_delay_secs: 60, gloas_fork_epoch: 0, log_filter: DEFAULT_LOG_FILTER.to_string(), }