diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb50e5dc8..232c865ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: # lag leanVM, and `--all-targets` leaves such targets out. Name them so # they keep compiling until they run again. - name: Clippy (skipped spectests) - run: cargo clippy --locked --workspace --test forkchoice_spectests --test signature_spectests --test stf_spectests --test ssz_spectests -- -D warnings + run: cargo clippy --locked --workspace --test forkchoice_spectests --test signature_spectests --test stf_spectests --test ssz_spectests --test interchange_vectors -- -D warnings # Two features gate code that no step in `lint` reaches, so both would rot # silently between merges. diff --git a/.gitignore b/.gitignore index b2e487fa2..cfaaffa96 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,10 @@ target # BLS and KZG test vectors, downloaded by `make cryptography-specs` /cryptography-specs +# EIP-3076 interchange test vectors, downloaded by +# `make slashing-protection-interchange-tests` +/slashing-protection-interchange-tests + # Personal Claude Code overrides (not shared with team) CLAUDE.local.md diff --git a/CLAUDE.md b/CLAUDE.md index 7d3f4d541..3584a1b3c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -879,8 +879,10 @@ transitions are in `ethlambda-types`, per the section above. Nothing above aggregate when `verified_sync_aggregate` fails (see `docs/spec_deviations.md`). `SlotClock` picks the offsets by each slot's own fork, and `/eth/v1/config/spec` supplies them. It - still keeps no slashing-protection record; PTC votes have an in-memory - `(validator, slot)` dedup only, and envelopes are unguarded. + keeps EIP-3076 minimal-watermark slashing protection in + `--validators-dir/slashing_protection.json` (`crate::slashing_protection`), + durable before every block and attestation signature; PTC votes have an + in-memory `(validator, slot)` dedup only, and envelopes are unguarded. `get_head_node` is one bottom-up walk for every fork, with the current slot's fork selecting the payload rules and the boost gate, so a follower on any network that schedules gloas (Sepolia does) pays work diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df164ea5c..78d970e15 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,7 +150,7 @@ If you discover a security vulnerability, **do not open a public issue**. Instea ## Communication -- **Telegram**: [ethlambda group](https://t.me/lambdaconsensus), where we post daily updates; drop by to ask questions or chat about anything Lean-related. +- **Telegram**: [ethlambda group](https://t.me/ethlambda), where we post daily updates; drop by to ask questions or chat about anything Lean-related. - **X (Twitter)**: [@ethlambda_lean](https://twitter.com/ethlambda_lean) for occasional updates. - **Weekly community call**: every Friday, streamed live on [@class_lambda](https://x.com/class_lambda); the call link is posted on Telegram beforehand. - **GitHub Issues**: bugs, feature requests, and technical discussion. diff --git a/Cargo.lock b/Cargo.lock index aeda51e1b..938463b32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2130,6 +2130,7 @@ dependencies = [ "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "rand 0.8.6", + "rayon", "secp256k1 0.30.0", "sha2", "snap", diff --git a/Makefile b/Makefile index 10796aff3..5fead8535 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help fmt lint bench update cooldown-check docker-build shadow-build shadow-docker-build run-devnet test test-consensus test-node test-beacon test-beacon-mainnet test-beacon-minimal consensus-spec-tests cryptography-specs docs docs-deps docs-serve +.PHONY: help fmt lint bench update cooldown-check docker-build shadow-build shadow-docker-build run-devnet test test-consensus test-node test-beacon test-beacon-mainnet test-beacon-minimal consensus-spec-tests cryptography-specs docs docs-deps docs-serve slashing-protection-interchange-tests help: ## ๐Ÿ“š Show help for each of the Makefile recipes @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -10,7 +10,7 @@ lint: ## ๐Ÿ” Run clippy on all workspace crates cargo clippy --locked --workspace --all-targets -- -D warnings # The spectests are `test = false` (stale leanSpec fixtures), which # `--all-targets` skips: lint them by name so they keep compiling - cargo clippy --locked --workspace --test forkchoice_spectests --test signature_spectests --test stf_spectests --test ssz_spectests -- -D warnings + cargo clippy --locked --workspace --test forkchoice_spectests --test signature_spectests --test stf_spectests --test ssz_spectests --test interchange_vectors -- -D warnings # release-fast: release-grade opt-level to avoid stack overflows during # signature verification/aggregation, without paying for LTO on every rebuild @@ -38,8 +38,9 @@ test: test-consensus test-node ## ๐Ÿงช Run all tests test-consensus: leanSpec/fixtures ## ๐Ÿงช Run the consensus half of the workspace suite $(TEST) $(addprefix -p ,$(CONSENSUS_CRATES)) -test-node: leanSpec/fixtures ## ๐Ÿงช Run the node half of the workspace suite +test-node: leanSpec/fixtures slashing-protection-interchange-tests ## ๐Ÿงช Run the node half of the workspace suite $(TEST) --workspace $(addprefix --exclude ,$(CONSENSUS_CRATES)) + $(TEST) -p ethlambda-validator --test interchange_vectors # --lib as well as the spec target: the BLS and KZG modules keep their fixture # vectors as unit tests, which are `ignore`d unless this feature is on. @@ -211,6 +212,33 @@ $(CRYPTOGRAPHY_SPECS_STAMP): unzip -q "$$tmpdir/tests.zip" -d cryptography-specs || exit 1 @touch $@ +# EIP-3076 slashing-protection interchange test vectors, for the validator +# client's `slashing_protection` module (crates/validator/tests/ +# interchange_vectors.rs). Pinned, and stamped by version like the fixture trees +# above, so a bump re-downloads rather than passing against the old cases. The +# release ships no asset for this tag, so the tag's source archive is fetched +# and checked against a pinned SHA-256 instead. +SLASHING_PROTECTION_TESTS_VERSION ?= v5.3.0 +SLASHING_PROTECTION_TESTS_SHA256 ?= 516d551cfb3e50e4ac2f42db0992f4ceb573a7cb1616d727a725c8161485329f +SLASHING_PROTECTION_TESTS_URL ?= https://github.com/eth-clients/slashing-protection-interchange-tests/archive/refs/tags/$(SLASHING_PROTECTION_TESTS_VERSION).tar.gz +SLASHING_PROTECTION_TESTS_STAMP=slashing-protection-interchange-tests/.version-$(SLASHING_PROTECTION_TESTS_VERSION) + +slashing-protection-interchange-tests: $(SLASHING_PROTECTION_TESTS_STAMP) ## โฌ‡๏ธ Download the EIP-3076 interchange test vectors + +$(SLASHING_PROTECTION_TESTS_STAMP): + @rm -rf slashing-protection-interchange-tests + @mkdir -p slashing-protection-interchange-tests + @tmpdir=$$(mktemp -d); trap 'rm -rf "$$tmpdir"' EXIT; \ + echo "Downloading EIP-3076 interchange test vectors ($(SLASHING_PROTECTION_TESTS_VERSION))"; \ + curl -L -f -o "$$tmpdir/tests.tar.gz" "$(SLASHING_PROTECTION_TESTS_URL)" || exit 1; \ + actual=$$(sha256sum "$$tmpdir/tests.tar.gz" | awk '{print $$1}'); \ + if [ "$$actual" != "$(SLASHING_PROTECTION_TESTS_SHA256)" ]; then \ + echo "SHA256 mismatch: expected $(SLASHING_PROTECTION_TESTS_SHA256), got $$actual" >&2; \ + exit 1; \ + fi; \ + tar -xzf "$$tmpdir/tests.tar.gz" -C slashing-protection-interchange-tests --strip-components=1 || exit 1 + @touch $@ + # lambdaclass fork of lean-quickstart: genesis keys come from `ethlambda keygen`, and the # partner clients run their devnet-5 images. An existing lean-quickstart/ is never # re-cloned, so delete it to pick up a new pin. diff --git a/README.md b/README.md index ff41a6a28..5cc44ef2c 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,7 @@ cargo build --release ``` [`docs/cli.md`](./docs/cli.md) is the full flag reference for every sub-command, -`benchmark` and `replay-era` included. The +`benchmark`, `replay-era` and `slashing-protection` included. The [quickstart](#beacon-chain-follower) runs a release binary with the same arguments. #### `ethlambda node` โ€” the Lean consensus chain @@ -341,13 +341,17 @@ payload envelope and the payload timeliness vote. It talks to a beacon node over the standard Beacon API, so it works against `ethlambda beacon` (run with the `--execution-*` flags) or any other conformant beacon node. -> **Warning:** the validator client keeps no slashing-protection record yet. -> Slashing protection is a work in progress and will be added soon. Until then, -> do not run the same keys in another client while this one runs, and treat a -> restart with the same care as a manual key move. Read -> [Spec deviations](./docs/spec_deviations.md#the-validator-client-keeps-no-slashing-protection-record) -> before using keys that hold real stake. The client repeats this warning at -> startup. +> **Slashing protection:** every block and attestation is checked against a +> durable per-validator record in `--validators-dir` before it is signed, and a +> second validator client pointed at the same directory refuses to start. A key +> with no record is refused at startup: pass `--init-slashing-protection` once, +> and only for keys that have signed nothing anywhere for at least two epochs; +> even then it rules out a double vote, not a surround of an older vote, which +> only the key's own history can. +> A key moving from another client brings its history with +> `ethlambda slashing-protection import`, or in the same keymanager API request +> that imports the key, and never runs in two clients at once. See +> [Spec deviations](./docs/spec_deviations.md#the-validator-client-keeps-eip-3076s-minimal-watermarks-not-full-history). ```sh ./target/release/ethlambda validator \ @@ -496,7 +500,7 @@ We have a proof-of-concept formalization of a part of the state transition funct ## Community -- **Telegram**: [ethlambda group](https://t.me/lambdaconsensus), where we post daily updates; drop by to ask questions or chat about anything Lean-related. +- **Telegram**: [ethlambda group](https://t.me/ethlambda), where we post daily updates; drop by to ask questions or chat about anything Lean-related. - **X (Twitter)**: [@ethlambda_lean](https://twitter.com/ethlambda_lean) for occasional updates. - **Weekly community call**: every Friday, streamed live on [@class_lambda](https://x.com/class_lambda); the call link is posted on Telegram beforehand. - **Ecosystem coordination**: the [PQ Interop calls](https://github.com/ethereum/pm/issues?q=is%3Aissue+%22PQ+Interop%22+in%3Atitle) on `ethereum/pm` cover cross-client Lean Ethereum work and related updates; the meeting links are posted on each issue. diff --git a/bin/ethlambda/src/benchmark/import/replay.rs b/bin/ethlambda/src/benchmark/import/replay.rs index e7bdf2794..21f936059 100644 --- a/bin/ethlambda/src/benchmark/import/replay.rs +++ b/bin/ethlambda/src/benchmark/import/replay.rs @@ -199,6 +199,14 @@ pub(crate) async fn replay_corpus(dir: &Path, options: &ReplayOptions) -> eyre:: // The same refusal startup makes, so a gloas corpus fails with its real // reason rather than partway through replay. crate::refuse_unfollowable_fork(anchor_state.fork_name())?; + // Mirrors what node startup does on its head state, so the first imports + // do not pay for validating the registry's keys. + let warm_started = Instant::now(); + let warmed = ethlambda_blockchain::warm_registry_pubkeys(&anchor_state); + eprintln!( + "registry warm-up: {warmed} keys validated in {:?}", + warm_started.elapsed() + ); let store = fork_choice::get_forkchoice_store(backend, anchor_state, anchor_block, &config)?; // A clone, not a borrow: `Store::set_time_ms` writes to the shared // backend's metadata, so this clone's write is the write every other diff --git a/bin/ethlambda/src/command.rs b/bin/ethlambda/src/command.rs index 7d0b9556a..d5cfe4d54 100644 --- a/bin/ethlambda/src/command.rs +++ b/bin/ethlambda/src/command.rs @@ -30,6 +30,7 @@ const EXPLICIT: &[&str] = &[ BENCHMARK, REPLAY_ERA, VALIDATOR, + SLASHING_PROTECTION, KEYGEN, "help", "-h", @@ -43,6 +44,7 @@ const BEACON: &str = "beacon"; const BENCHMARK: &str = "benchmark"; const REPLAY_ERA: &str = "replay-era"; const VALIDATOR: &str = "validator"; +const SLASHING_PROTECTION: &str = "slashing-protection"; const KEYGEN: &str = "keygen"; #[derive(Debug, clap::Parser)] @@ -84,6 +86,9 @@ pub(crate) enum Command { ReplayEra(ReplayEraOptions), /// Run the beacon-chain validator client against a beacon node's HTTP API. Validator(crate::validator::ValidatorOptions), + /// Import or export the validator client's EIP-3076 slashing-protection + /// history, offline. + SlashingProtection(crate::slashing_protection::SlashingProtectionOptions), /// Generate validator XMSS keys for a genesis. Keygen(KeygenOptions), } @@ -415,7 +420,14 @@ mod tests { /// `ethlambda node keygen ...`, which fails on a stray positional. #[test] fn sub_command_tokens_do_not_get_the_default_inserted() { - for token in [NODE, BEACON, BENCHMARK, VALIDATOR, KEYGEN] { + for token in [ + NODE, + BEACON, + BENCHMARK, + VALIDATOR, + SLASHING_PROTECTION, + KEYGEN, + ] { let args = [OsString::from("ethlambda"), OsString::from(token)]; assert_eq!( default_subcommand(&args), @@ -425,6 +437,30 @@ mod tests { } } + #[test] + fn slashing_protection_parses_as_its_own_sub_command() { + let args = [ + "ethlambda", + SLASHING_PROTECTION, + "export", + "--validators-dir", + "/tmp/validators", + "--pubkeys", + "0xaa,0xbb", + "out.json", + ]; + match try_parse_from(args.iter().map(OsString::from)).expect("parses") { + Command::SlashingProtection(options) => match options.action { + crate::slashing_protection::Action::Export { pubkeys, file, .. } => { + assert_eq!(pubkeys, vec!["0xaa", "0xbb"]); + assert_eq!(file, PathBuf::from("out.json")); + } + other => panic!("expected an export, got {other:?}"), + }, + other => panic!("expected a slashing-protection invocation, got {other:?}"), + } + } + #[test] fn keygen_parses_as_its_own_sub_command() { let args = [ diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index b9e86dfa5..2271de672 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -8,6 +8,7 @@ mod era; mod fd_limit; mod keygen; mod network; +mod slashing_protection; mod validator; mod version; @@ -109,6 +110,11 @@ fn main() -> eyre::Result<()> { init_node_logging()?; validator::run(options) } + // Offline file work: no runtime, and stdout left alone. + Command::SlashingProtection(options) => { + init_info_stderr_logging()?; + slashing_protection::run(options) + } // Key generation is synchronous, CPU-bound work whose product is files, // so it runs on this thread too and leaves stdout alone. Command::Keygen(options) => { diff --git a/bin/ethlambda/src/slashing_protection.rs b/bin/ethlambda/src/slashing_protection.rs new file mode 100644 index 000000000..044e51250 --- /dev/null +++ b/bin/ethlambda/src/slashing_protection.rs @@ -0,0 +1,137 @@ +//! `ethlambda slashing-protection`: move a validator client's EIP-3076 history +//! in and out of its `--validators-dir`. +//! +//! Offline on purpose. It takes the same lock the validator client holds while +//! it runs, so it refuses while one is running there: history imported under a +//! running client could arrive after the client had already signed past it. +//! No beacon node is needed either: the record names its own chain, and an +//! import into a directory with no record yet creates one for the chain the +//! interchange names, which the client checks against its beacon node at its +//! next start. + +use std::io::Write as _; +use std::path::PathBuf; + +use ethlambda_validator::beacon_node::dto::{encode_hex, parse_pubkey}; +use ethlambda_validator::slashing_protection::SlashingProtection; +use tracing::info; + +#[derive(Debug, clap::Args)] +pub(crate) struct SlashingProtectionOptions { + #[command(subcommand)] + pub(crate) action: Action, +} + +#[derive(Debug, clap::Subcommand)] +pub(crate) enum Action { + /// Merge an EIP-3076 interchange file (complete or minimal, version 5) into + /// the record, never lowering what is already there. + Import { + /// The validator client's `--validators-dir`. + #[arg(long)] + validators_dir: PathBuf, + /// The interchange file, as exported by this or another client. + file: PathBuf, + }, + /// Write the record as a minimal EIP-3076 interchange file (version 5). + Export { + /// The validator client's `--validators-dir`. + #[arg(long)] + validators_dir: PathBuf, + /// Only these validators, `0x`-prefixed, comma-separated or repeated. + /// Every validator with a record when omitted. + #[arg(long, value_delimiter = ',')] + pubkeys: Vec, + /// Where to write it. Refused if it exists, rather than overwritten. + file: PathBuf, + }, +} + +pub(crate) fn run(options: SlashingProtectionOptions) -> eyre::Result<()> { + match options.action { + Action::Import { + validators_dir, + file, + } => { + let interchange = std::fs::read_to_string(&file) + .map_err(|err| eyre::eyre!("{}: {err}", file.display()))?; + // Parsed once: the file can be tens of megabytes in the complete + // format, and its chain is needed before the record is opened. + let staged = SlashingProtection::stage(&interchange)?; + let mut protection = + SlashingProtection::open_offline(&validators_dir, Some(staged.root()))?; + let validators = protection.absorb(Some(&staged), &[])?.named; + info!( + validators, + file = %file.display(), + "Imported slashing-protection history" + ); + } + Action::Export { + validators_dir, + pubkeys, + file, + } => { + let protection = SlashingProtection::open_offline(&validators_dir, None)?; + let pubkeys = if pubkeys.is_empty() { + protection.pubkeys() + } else { + pubkeys + .iter() + .map(|text| { + parse_pubkey(text).map_err(|err| eyre::eyre!("--pubkeys {text}: {err}")) + }) + .collect::, _>>()? + }; + // A requested key with no record fails the export, as Lighthouse + // does, rather than being left out of a file that then reads as + // complete: a typo, or the wrong directory, would otherwise be + // found out at the other end, after the keys moved. + let missing: Vec = pubkeys + .iter() + .filter(|pubkey| !protection.contains(pubkey)) + .map(|pubkey| encode_hex(&pubkey.0)) + .collect(); + if !missing.is_empty() { + eyre::bail!( + "no slashing-protection record for {} requested validator(s) (first: {}); \ + nothing was written", + missing.len(), + missing[0] + ); + } + let exported = pubkeys.len(); + let interchange = protection.export(&pubkeys)?; + write_export(&file, &interchange) + .map_err(|err| eyre::eyre!("{}: {err}", file.display()))?; + info!( + validators = exported, + file = %file.display(), + "Exported slashing-protection history" + ); + } + } + Ok(()) +} + +/// Write `interchange` to a new file at `path`, refusing an existing one. +/// +/// Private to the owner, like the record it was taken from. A write that +/// fails partway removes what it left, so the retry is not refused for a file +/// this run created and never finished. +fn write_export(path: &std::path::Path, interchange: &str) -> std::io::Result<()> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + let mut output = options.open(path)?; + output + .write_all(interchange.as_bytes()) + .and_then(|()| output.sync_all()) + .inspect_err(|_| { + let _ = std::fs::remove_file(path); + }) +} diff --git a/bin/ethlambda/src/validator.rs b/bin/ethlambda/src/validator.rs index f97f82782..968244963 100644 --- a/bin/ethlambda/src/validator.rs +++ b/bin/ethlambda/src/validator.rs @@ -78,6 +78,19 @@ pub(crate) struct ValidatorOptions { /// beyond localhost must front it with TLS. #[arg(long, default_value = "false")] pub(crate) enable_keymanager: bool, + + /// Register every loaded key that has no slashing-protection record, + /// instead of refusing to start. + /// + /// DANGEROUS. Use it only for keys that are brand new, or that have signed + /// nothing anywhere for at least two epochs. A key is registered as never + /// having signed, so nothing here stops it from signing again what it + /// signed elsewhere a moment ago; and two idle epochs rule out a double + /// vote, not a surround of an older vote, which only the key's own history + /// can. Keys that already have a record keep it, so the flag never lowers + /// one; remove it once the keys are registered. + #[arg(long, default_value = "false")] + pub(crate) init_slashing_protection: bool, } impl ValidatorOptions { @@ -154,6 +167,7 @@ pub(crate) async fn run(options: ValidatorOptions) -> eyre::Result<()> { keymanager: options .enable_keymanager .then(|| std::net::SocketAddr::new(options.http_address, options.keymanager_port)), + init_slashing_protection: options.init_slashing_protection, }) .await .map_err(Into::into) @@ -174,6 +188,7 @@ mod tests { graffiti: graffiti.to_string(), suggested_fee_recipient: None, enable_keymanager: false, + init_slashing_protection: false, } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index f59dd5c6e..3fdd2cf60 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -16,7 +16,9 @@ use ethlambda_types::{ beacon::{ config::Config, constants, - containers::{DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, gloas}, + containers::{ + BeaconState, DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, gloas, + }, preset, primitives::ValidatorIndex, signing::compute_epoch_at_slot, @@ -79,6 +81,34 @@ pub mod spec_test_runner; pub mod store; mod sync_status; +/// Validates every public key in `state`'s registry in one parallel pass and +/// returns how many passed. +/// +/// Run when a state is loaded to run on, so the first imported block does not +/// validate tens of thousands of keys inline. Keys are cached with their +/// registry index, which is how the state transition looks them up. +pub fn warm_registry_pubkeys(state: &BeaconState) -> usize { + let started = Instant::now(); + let pubkeys: Vec = state + .iter_validators() + .enumerate() + .map(|(index, validator)| ethlambda_crypto::bls::IndexedPubkey { + index: index as ValidatorIndex, + pubkey: validator.pubkey, + }) + .collect(); + let registry_size = pubkeys.len(); + let count = ethlambda_crypto::bls::warm_pubkeys(&pubkeys); + let elapsed = started.elapsed(); + info!( + count, + registry_size, + ?elapsed, + "Registry public keys validated" + ); + count +} + pub struct BlockChain { handle: ActorRef, } @@ -359,6 +389,15 @@ impl BlockChain { metrics::set_node_sync_status(metrics::SyncStatus::Idle); + // Before the actor can import anything, so the first block does not + // validate the registry's keys inline. + match store.head().map(|head| store.get_state(&head)) { + Ok(Ok(Some(state))) => { + warm_registry_pubkeys(&state); + } + _ => warn!("Head state unavailable, skipping the registry key warm-up"), + } + Self::start_actor( store, SyncStatusTracker::new(false), diff --git a/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs b/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs index 3b7feaa1a..00bd8e5bb 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs @@ -56,8 +56,8 @@ use std::num::NonZeroUsize; use lru::LruCache; use super::{ - IgnoreReason, Outcome, RejectReason, ancestor_at, is_current_or_previous_epoch, is_future_slot, - verify_attestation_payload_status, + IgnoreReason, Outcome, Prepared, RejectReason, ancestor_at, is_current_or_previous_epoch, + is_future_slot, verify_attestation_payload_status, }; use crate::beacon::bls; use crate::beacon::constants::{ @@ -364,28 +364,45 @@ pub fn stateful_checks( store: &Store, aggregate: &SignedAggregateAndProof, ) -> Result, Outcome> { + prepare_stateful_checks(store, aggregate).resolve() +} + +/// [`stateful_checks`] without evaluating any pairing: a [`Prepared`] whose +/// steps are the rules in their original order (selection proof, aggregator +/// signature, the committee rules, the aggregate signature, ancestry), the +/// three signatures among them as deferred steps. +/// +/// Every non-signature rule runs here, ahead of the pairings, so the +/// committee cache is consulted even for an aggregate whose pubkey-only +/// signatures turn out forged. That is the price of letting the pairings of +/// many messages be batched; the verdict is unchanged, since resolution takes +/// the first failure in the original order. +pub fn prepare_stateful_checks( + store: &Store, + aggregate: &SignedAggregateAndProof, +) -> Prepared> { + let mut prepared = Prepared::default(); let data = aggregate.data(); let beacon_block_root = data.beacon_block_root; // [IGNORE] The block being voted for has been seen. if !store.has_block(&beacon_block_root) { - return Err(Outcome::Ignore(IgnoreReason::UnknownBlock)); + return prepared.fail(Outcome::Ignore(IgnoreReason::UnknownBlock)); } - // See this function's own documentation for why this state, and the + // See `stateful_checks`'s own documentation for why this state, and the // module documentation for why an uncached state is `IGNORE` rather than // the specification's `REJECT`. let Some(state) = store.cached_state(CacheKey::BlockState(beacon_block_root)) else { - return Err(Outcome::Ignore(IgnoreReason::StateUnavailable)); + return prepared.fail(Outcome::Ignore(IgnoreReason::StateUnavailable)); }; let target_epoch = data.target.epoch; let config = store.config(); - // Pubkey-only signatures, before any committee derivation; see the - // module documentation for why this order. + // Pubkey-only signatures; see the module documentation. let aggregator_index = aggregate.aggregator_index(); let Ok(aggregator) = state.validator(aggregator_index) else { - return Err(Outcome::Reject(RejectReason::UnknownValidator)); + return prepared.fail(Outcome::Reject(RejectReason::UnknownValidator)); }; let aggregator_pubkey = bls::IndexedPubkey { index: aggregator_index, @@ -396,21 +413,23 @@ pub fn stateful_checks( let selection_domain = get_domain_from_schedule(&config, &state, DOMAIN_SELECTION_PROOF, target_epoch); let selection_signing_root = compute_signing_root(data.slot.hash_tree_root(), selection_domain); - if !bls::verify(&aggregator_pubkey, selection_signing_root, &selection_proof) { - return Err(Outcome::Reject(RejectReason::SelectionProof)); - } + prepared.signature( + bls::SignatureSet::single(&aggregator_pubkey, selection_signing_root, &selection_proof), + Outcome::Reject(RejectReason::SelectionProof), + ); // [REJECT] The aggregator's own signature, over the whole envelope. let aggregator_domain = get_domain_from_schedule(&config, &state, DOMAIN_AGGREGATE_AND_PROOF, target_epoch); let aggregator_signing_root = compute_signing_root(aggregate_and_proof_root(aggregate), aggregator_domain); - if !bls::verify( - &aggregator_pubkey, - aggregator_signing_root, - &aggregate.signature(), - ) { - return Err(Outcome::Reject(RejectReason::AggregatorSignature)); - } + prepared.signature( + bls::SignatureSet::single( + &aggregator_pubkey, + aggregator_signing_root, + &aggregate.signature(), + ), + Outcome::Reject(RejectReason::AggregatorSignature), + ); // Committees for the target epoch, through the shared cache. let committees = store.committee_cache(); @@ -421,115 +440,120 @@ pub fn stateful_checks( .expect("cheap_checks rejects a message with no single named committee"); // [REJECT] The committee index is within range. if committee_index >= epoch_committees.committees_per_slot() { - return Err(Outcome::Reject(RejectReason::CommitteeIndex)); + return prepared.fail(Outcome::Reject(RejectReason::CommitteeIndex)); } let Ok(committee) = epoch_committees.committee(data.slot, committee_index) else { // `committee_index` and `data.slot`'s epoch were both just checked // against this same `epoch_committees`, so this cannot fail; treated // as internal rather than unreachable so a future change here fails // safe instead of panicking. - return Err(Outcome::Ignore(IgnoreReason::Internal)); + return prepared.fail(Outcome::Ignore(IgnoreReason::Internal)); }; // [REJECT] The aggregation bits match the committee's size. let aggregation_bits = aggregate.aggregation_bits(); if aggregation_bits.len() != committee.len() { - return Err(Outcome::Reject(RejectReason::BitsLength)); + return prepared.fail(Outcome::Reject(RejectReason::BitsLength)); } // [REJECT] The selection proof selects this validator as an aggregator. if !is_aggregator(committee.len(), &selection_proof) { - return Err(Outcome::Reject(RejectReason::NotAggregator)); + return prepared.fail(Outcome::Reject(RejectReason::NotAggregator)); } // [REJECT] The aggregator is a member of the committee. if !committee.contains(&aggregator_index) { - return Err(Outcome::Reject(RejectReason::NotInCommittee)); + return prepared.fail(Outcome::Reject(RejectReason::NotInCommittee)); } // [REJECT] The aggregate's own signature is valid. Built from the same // (cached) committees, so this costs no further shuffle. let attester_domain = get_domain_from_schedule(&config, &state, DOMAIN_BEACON_ATTESTER, target_epoch); - let attesting_indices = match aggregate { + let (attesting_indices, aggregate_set) = match aggregate { SignedAggregateAndProof::Phase0(signed) => { let phase0_attestation = &signed.message.aggregate; - let indexed = crate::beacon::helpers::attestation::get_indexed_attestation( + let Ok(indexed) = crate::beacon::helpers::attestation::get_indexed_attestation( &state, phase0_attestation, &committees, - ) - .map_err(|_| Outcome::Ignore(IgnoreReason::Internal))?; - if !crate::beacon::helpers::attestation::is_valid_indexed_attestation_with_domain( - &state, - &indexed, - attester_domain, - ) { - return Err(Outcome::Reject(RejectReason::AggregateSignature)); - } - indexed.attesting_indices.to_vec() + ) else { + return prepared.fail(Outcome::Ignore(IgnoreReason::Internal)); + }; + let set = + crate::beacon::helpers::attestation::indexed_attestation_signature_set_with_domain( + &state, + &indexed, + attester_domain, + ); + (indexed.attesting_indices.to_vec(), set) } SignedAggregateAndProof::Electra(signed) => { let electra_attestation = &signed.message.aggregate; - let indexed = crate::beacon::helpers::electra::get_indexed_attestation( + let Ok(indexed) = crate::beacon::helpers::electra::get_indexed_attestation( &state, electra_attestation, &committees, - ) - .map_err(|_| Outcome::Ignore(IgnoreReason::Internal))?; - if !crate::beacon::helpers::electra::is_valid_indexed_attestation_with_domain( - &state, - &indexed, - attester_domain, - ) { - return Err(Outcome::Reject(RejectReason::AggregateSignature)); - } - indexed.attesting_indices.to_vec() + ) else { + return prepared.fail(Outcome::Ignore(IgnoreReason::Internal)); + }; + let set = + crate::beacon::helpers::electra::indexed_attestation_signature_set_with_domain( + &state, + &indexed, + attester_domain, + ); + (indexed.attesting_indices.to_vec(), set) } SignedAggregateAndProof::Gloas(signed) => { let gloas_attestation = &signed.message.aggregate; - let indexed = crate::beacon::helpers::gloas::get_indexed_attestation( + let Ok(indexed) = crate::beacon::helpers::gloas::get_indexed_attestation( &state, gloas_attestation, &committees, - ) - .map_err(|_| Outcome::Ignore(IgnoreReason::Internal))?; - if !crate::beacon::helpers::gloas::is_valid_indexed_attestation_with_domain( + ) else { + return prepared.fail(Outcome::Ignore(IgnoreReason::Internal)); + }; + let set = crate::beacon::helpers::gloas::indexed_attestation_signature_set_with_domain( &state, &indexed, attester_domain, - ) { - return Err(Outcome::Reject(RejectReason::AggregateSignature)); - } - indexed.attesting_indices.to_vec() + ); + (indexed.attesting_indices.to_vec(), set) } }; + prepared.signature( + aggregate_set, + Outcome::Reject(RejectReason::AggregateSignature), + ); - // Ancestry, via the vote state's own history; see this function's + // Ancestry, via the vote state's own history; see `stateful_checks`'s // documentation for why this state can always answer both questions. let (target_checkpoint_epoch, target_root) = aggregate.target(); let target_start_slot = compute_start_slot_at_epoch(target_checkpoint_epoch); // [REJECT] The target is the vote block's ancestor at the target epoch. let Some(checkpoint_block) = ancestor_at(&state, beacon_block_root, target_start_slot) else { - return Err(Outcome::Ignore(IgnoreReason::AncestryUnknown)); + return prepared.fail(Outcome::Ignore(IgnoreReason::AncestryUnknown)); }; if checkpoint_block != target_root { - return Err(Outcome::Reject(RejectReason::TargetNotAncestor)); + return prepared.fail(Outcome::Reject(RejectReason::TargetNotAncestor)); } // [IGNORE] The finalized checkpoint is an ancestor of the vote block. let finalized = store.beacon_finalized_checkpoint(); let finalized_start_slot = compute_start_slot_at_epoch(finalized.epoch); let Some(finalized_block) = ancestor_at(&state, beacon_block_root, finalized_start_slot) else { - return Err(Outcome::Ignore(IgnoreReason::AncestryUnknown)); + return prepared.fail(Outcome::Ignore(IgnoreReason::AncestryUnknown)); }; if finalized_block != finalized.root { - return Err(Outcome::Ignore(IgnoreReason::FinalizedNotAncestor)); + return prepared.fail(Outcome::Ignore(IgnoreReason::FinalizedNotAncestor)); } // [New in Gloas:EIP7732] The attested payload status is consistent with the // block's execution payload. - if matches!(aggregate, SignedAggregateAndProof::Gloas(_)) { - verify_attestation_payload_status(store, &data)?; + if matches!(aggregate, SignedAggregateAndProof::Gloas(_)) + && let Err(outcome) = verify_attestation_payload_status(store, &data) + { + return prepared.fail(outcome); } - Ok(attesting_indices) + prepared.accept(attesting_indices) } /// `cheap_checks` then `stateful_checks`, the order the p2p actor runs them @@ -900,4 +924,221 @@ mod tests { fn preset_slots_per_epoch() -> Slot { crate::beacon::preset::SLOTS_PER_EPOCH } + + // -- prepare_stateful_checks --------------------------------------------- + + /// What to get wrong in [`signed_aggregate`]. + #[derive(Default)] + struct Mistakes { + /// The selection proof is another validator's. + selection_proof: bool, + /// The attesters' aggregate signature is another message's. + aggregate_signature: bool, + /// `target.root` names a block that is not the vote block's ancestor. + target_root: bool, + /// `committee_bits` names a committee the epoch does not have. + committee_bit: bool, + } + + /// A store holding the vote block's state, and an aggregate for the first + /// committee of the first slot of the fork's epoch, signed correctly except + /// where `mistakes` says. Mirrors the attestation tests' fork-boundary + /// fixture, whose signing domain is the schedule's, not the state's. + fn signed_aggregate(mistakes: Mistakes) -> (Store, SignedAggregateAndProof) { + use crate::beacon::config::Config; + use crate::beacon::containers::SignedBeaconBlock; + use crate::beacon::containers::shared::{AttestationData, Checkpoint, Fork}; + use crate::beacon::fork::ForkName; + use crate::beacon::gossip::test_support::store_with_config; + use crate::beacon::helpers::accessors::get_beacon_committee; + use crate::beacon::helpers::misc::compute_domain; + use crate::beacon::helpers::test_state::{sign_for, with_signing_validators_at}; + + let fulu_epoch = 2; + let config = Config::mainnet() + .with_fork_epoch(ForkName::Electra, 0) + .with_fork_epoch(ForkName::Fulu, fulu_epoch); + let mut state = with_signing_validators_at(ForkName::Electra, 64); + let pre_fork_slot = compute_start_slot_at_epoch(fulu_epoch) - 1; + *state.slot_mut() = pre_fork_slot; + *state.fork_mut() = Fork { + previous_version: config.deneb_fork_version, + current_version: config.electra_fork_version, + epoch: 0, + }; + state.apply_pending_mutations(); + + let mut store = store_with_config(0, config.clone()); + let block_root = Root::repeat_byte(7); + let block = SignedBeaconBlock::Electra(electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot: pre_fork_slot, + proposer_index: 0, + parent_root: Root::ZERO, + state_root: Root::ZERO, + body: electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }); + store + .insert_pending_block(block_root, block) + .expect("insert the voted block"); + store.cache_state( + CacheKey::BlockState(block_root), + std::sync::Arc::new(state.clone()), + ); + + let slot = compute_start_slot_at_epoch(fulu_epoch); + let committee = get_beacon_committee(&state, slot, 0).expect("committee"); + let aggregator = committee[0]; + let data = AttestationData { + slot, + index: 0, + beacon_block_root: block_root, + source: Default::default(), + target: Checkpoint { + epoch: fulu_epoch, + root: if mistakes.target_root { + Root::repeat_byte(9) + } else { + block_root + }, + }, + }; + let domain = |domain_type| { + compute_domain( + domain_type, + config.fulu_fork_version, + state.genesis_validators_root(), + ) + }; + let attester_root = + compute_signing_root(data.hash_tree_root(), domain(DOMAIN_BEACON_ATTESTER)); + let signatures: Vec = committee + .iter() + .map(|&member| { + let message = if mistakes.aggregate_signature { + Root::repeat_byte(5) + } else { + attester_root + }; + sign_for(member as usize, message) + }) + .collect(); + let selector = if mistakes.selection_proof { + aggregator as usize + 1 + } else { + aggregator as usize + }; + let selection_proof = sign_for( + selector, + compute_signing_root(slot.hash_tree_root(), domain(DOMAIN_SELECTION_PROOF)), + ); + let mut committee_bits = electra::CommitteeBits::default(); + let bit = if mistakes.committee_bit { + crate::beacon::preset::MAX_COMMITTEES_PER_SLOT - 1 + } else { + 0 + }; + committee_bits.set(bit, true).expect("within the bound"); + let message = electra::AggregateAndProof { + aggregator_index: aggregator, + aggregate: electra::Attestation { + aggregation_bits: vec![true; committee.len()] + .try_into() + .expect("within the committee bound"), + data, + signature: bls::aggregate(&signatures).expect("aggregate"), + committee_bits, + }, + selection_proof, + }; + let envelope_root = + compute_signing_root(message.hash_tree_root(), domain(DOMAIN_AGGREGATE_AND_PROOF)); + let signed = electra::SignedAggregateAndProof { + signature: sign_for(aggregator as usize, envelope_root), + message, + }; + (store, SignedAggregateAndProof::Electra(signed)) + } + + #[test] + fn a_valid_aggregate_prepares_three_pairings_and_resolves_either_way() { + let (store, aggregate) = signed_aggregate(Mistakes::default()); + let prepared = prepare_stateful_checks(&store, &aggregate); + let sets = prepared.signature_sets(); + assert_eq!(sets.len(), 3); + assert!(bls::verify_signature_sets(&sets)); + let batched = prepared.resolve_batch_passed().expect("accepted"); + assert!(!batched.is_empty()); + assert_eq!(stateful_checks(&store, &aggregate), Ok(batched)); + } + + /// The signatures come before the committee rules, so an aggregate that + /// breaks both reports the signature, as it did when each was verified in + /// place. + #[test] + fn a_bad_selection_proof_outranks_a_later_committee_failure() { + let (store, aggregate) = signed_aggregate(Mistakes { + selection_proof: true, + committee_bit: true, + ..Default::default() + }); + assert_eq!( + stateful_checks(&store, &aggregate), + Err(Outcome::Reject(RejectReason::SelectionProof)) + ); + } + + #[test] + fn a_good_selection_proof_lets_the_committee_failure_speak() { + let (store, aggregate) = signed_aggregate(Mistakes { + committee_bit: true, + ..Default::default() + }); + assert_eq!( + stateful_checks(&store, &aggregate), + Err(Outcome::Reject(RejectReason::CommitteeIndex)) + ); + } + + #[test] + fn a_bad_aggregate_signature_outranks_a_later_ancestry_failure() { + let (store, aggregate) = signed_aggregate(Mistakes { + aggregate_signature: true, + target_root: true, + ..Default::default() + }); + assert_eq!( + stateful_checks(&store, &aggregate), + Err(Outcome::Reject(RejectReason::AggregateSignature)) + ); + let (store, aggregate) = signed_aggregate(Mistakes { + target_root: true, + ..Default::default() + }); + assert_eq!( + stateful_checks(&store, &aggregate), + Err(Outcome::Reject(RejectReason::TargetNotAncestor)) + ); + } + + #[test] + fn a_failed_batch_resolves_each_aggregate_on_its_own() { + let (store, good) = signed_aggregate(Mistakes::default()); + let (_, forged) = signed_aggregate(Mistakes { + aggregate_signature: true, + ..Default::default() + }); + let good = prepare_stateful_checks(&store, &good); + let forged = prepare_stateful_checks(&store, &forged); + let mut sets = good.signature_sets(); + sets.extend(forged.signature_sets()); + assert!(!bls::verify_signature_sets(&sets)); + assert!(good.resolve().is_ok()); + assert_eq!( + forged.resolve(), + Err(Outcome::Reject(RejectReason::AggregateSignature)) + ); + } } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/attestation.rs b/crates/blockchain/state_transition/src/beacon/gossip/attestation.rs index 711d2acac..da74dab8d 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/attestation.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/attestation.rs @@ -44,8 +44,8 @@ use std::num::NonZeroUsize; use lru::LruCache; use super::{ - IgnoreReason, Outcome, RejectReason, ancestor_at, is_current_or_previous_epoch, is_future_slot, - is_gloas_slot, verify_attestation_payload_status, + IgnoreReason, Outcome, Prepared, RejectReason, ancestor_at, is_current_or_previous_epoch, + is_future_slot, is_gloas_slot, verify_attestation_payload_status, }; use crate::beacon::bls; use crate::beacon::config::Config; @@ -169,24 +169,45 @@ pub fn cheap_checks( /// vote block's own cached post-state rather than the head's, and why the /// pubkey-only signature check runs before any committee derivation. pub fn stateful_checks(store: &Store, attestation: &SingleAttestation, subnet_id: u64) -> Outcome { + match prepare_stateful_checks(store, attestation, subnet_id).resolve() { + Ok(()) => Outcome::Accept, + Err(outcome) => outcome, + } +} + +/// [`stateful_checks`] without evaluating the attester's signature: a +/// [`Prepared`] whose steps are the rules in their original order, the +/// signature among them. +/// +/// Every other rule runs here, ahead of the pairing, so the committee cache +/// is consulted even for a message whose signature will turn out forged. That +/// is the price of letting the pairings of many messages be batched; the +/// verdict is unchanged, since resolution takes the first failure in the +/// original order. +pub fn prepare_stateful_checks( + store: &Store, + attestation: &SingleAttestation, + subnet_id: u64, +) -> Prepared<()> { + let mut prepared = Prepared::default(); let data = &attestation.data; let beacon_block_root = data.beacon_block_root; // [IGNORE] The block being voted for has been seen. if !store.has_block(&beacon_block_root) { - return Outcome::Ignore(IgnoreReason::UnknownBlock); + return prepared.fail(Outcome::Ignore(IgnoreReason::UnknownBlock)); } let Some(state) = store.cached_state(CacheKey::BlockState(beacon_block_root)) else { - return Outcome::Ignore(IgnoreReason::StateUnavailable); + return prepared.fail(Outcome::Ignore(IgnoreReason::StateUnavailable)); }; let target_epoch = data.target.epoch; let config = store.config(); - // The pubkey-only signature, before any committee derivation, under the - // schedule's domain: the voted block's state may predate the target's fork. + // The pubkey-only signature, under the schedule's domain: the voted + // block's state may predate the target's fork. let Ok(attester) = state.validator(attestation.attester_index) else { - return Outcome::Reject(RejectReason::UnknownValidator); + return prepared.fail(Outcome::Reject(RejectReason::UnknownValidator)); }; let domain = get_domain_from_schedule(&config, &state, DOMAIN_BEACON_ATTESTER, target_epoch); let signing_root = compute_signing_root(data.hash_tree_root(), domain); @@ -194,16 +215,17 @@ pub fn stateful_checks(store: &Store, attestation: &SingleAttestation, subnet_id index: attestation.attester_index, pubkey: attester.pubkey, }; - if !bls::verify(&attester, signing_root, &attestation.signature) { - return Outcome::Reject(RejectReason::BadSignature); - } + prepared.signature( + bls::SignatureSet::single(&attester, signing_root, &attestation.signature), + Outcome::Reject(RejectReason::BadSignature), + ); // Committees for the target epoch, through the shared cache. let committees = store.committee_cache(); let epoch_committees = committees.committees(&state, target_epoch); // [REJECT] The committee index is within range. if attestation.committee_index >= epoch_committees.committees_per_slot() { - return Outcome::Reject(RejectReason::CommitteeIndex); + return prepared.fail(Outcome::Reject(RejectReason::CommitteeIndex)); } // [New in Electra:EIP7549] [REJECT] The correct subnet. let expected_subnet = compute_subnet_for_attestation( @@ -213,36 +235,36 @@ pub fn stateful_checks(store: &Store, attestation: &SingleAttestation, subnet_id &config, ); if expected_subnet != subnet_id { - return Outcome::Reject(RejectReason::WrongSubnet); + return prepared.fail(Outcome::Reject(RejectReason::WrongSubnet)); } let Ok(committee) = epoch_committees.committee(data.slot, attestation.committee_index) else { // `committee_index` and `data.slot`'s epoch were both just checked // against this same `epoch_committees`, so this cannot fail. - return Outcome::Ignore(IgnoreReason::Internal); + return prepared.fail(Outcome::Ignore(IgnoreReason::Internal)); }; // [New in Electra:EIP7549] [REJECT] The attester is a member of the // named committee. if !committee.contains(&attestation.attester_index) { - return Outcome::Reject(RejectReason::NotInCommittee); + return prepared.fail(Outcome::Reject(RejectReason::NotInCommittee)); } // Ancestry, via the vote state's own history. let target_start_slot = compute_start_slot_at_epoch(target_epoch); // [REJECT] The target is the vote block's ancestor at the target epoch. let Some(checkpoint_block) = ancestor_at(&state, beacon_block_root, target_start_slot) else { - return Outcome::Ignore(IgnoreReason::AncestryUnknown); + return prepared.fail(Outcome::Ignore(IgnoreReason::AncestryUnknown)); }; if checkpoint_block != data.target.root { - return Outcome::Reject(RejectReason::TargetNotAncestor); + return prepared.fail(Outcome::Reject(RejectReason::TargetNotAncestor)); } // [IGNORE] The finalized checkpoint is an ancestor of the vote block. let finalized = store.beacon_finalized_checkpoint(); let finalized_start_slot = compute_start_slot_at_epoch(finalized.epoch); let Some(finalized_block) = ancestor_at(&state, beacon_block_root, finalized_start_slot) else { - return Outcome::Ignore(IgnoreReason::AncestryUnknown); + return prepared.fail(Outcome::Ignore(IgnoreReason::AncestryUnknown)); }; if finalized_block != finalized.root { - return Outcome::Ignore(IgnoreReason::FinalizedNotAncestor); + return prepared.fail(Outcome::Ignore(IgnoreReason::FinalizedNotAncestor)); } // [New in Gloas:EIP7732] The attested payload status is consistent with the @@ -250,10 +272,10 @@ pub fn stateful_checks(store: &Store, attestation: &SingleAttestation, subnet_id if is_gloas_slot(&store.config(), data.slot) && let Err(outcome) = verify_attestation_payload_status(store, data) { - return outcome; + return prepared.fail(outcome); } - Outcome::Accept + prepared.accept(()) } /// `cheap_checks` then `stateful_checks`, for callers with no reason to @@ -442,8 +464,10 @@ mod tests { /// A vote cast in a fork's first slot while that slot has no block: the /// voted block, and so the state it is checked against, is still the /// previous fork's, but the vote is signed under the new fork's version. - #[test] - fn a_vote_across_a_fork_boundary_verifies_under_the_new_forks_version() { + /// + /// Returns the store holding the voted block's state, the correctly signed + /// vote, and the subnet it belongs on. + fn signed_vote_across_a_fork_boundary() -> (Store, SingleAttestation, u64) { use crate::beacon::containers::shared::{AttestationData, Checkpoint, Fork}; use crate::beacon::containers::{SignedBeaconBlock, electra}; use crate::beacon::fork::ForkName; @@ -518,10 +542,107 @@ mod tests { .committees(&state, fulu_epoch) .committees_per_slot(); let subnet_id = compute_subnet_for_attestation(committees_per_slot, slot, 0, &config); + (store, attestation, subnet_id) + } + #[test] + fn a_vote_across_a_fork_boundary_verifies_under_the_new_forks_version() { + let (store, attestation, subnet_id) = signed_vote_across_a_fork_boundary(); assert_eq!( stateful_checks(&store, &attestation, subnet_id), Outcome::Accept ); } + + // -- prepare_stateful_checks --------------------------------------------- + + /// `attestation` with its signature replaced by another validator's over + /// the same data: well-formed, so it fails the pairing and not the decode. + fn with_a_wrong_signature(mut attestation: SingleAttestation) -> SingleAttestation { + use crate::beacon::helpers::test_state::sign_for; + let other = attestation.attester_index as usize + 1; + attestation.signature = sign_for(other, attestation.data.hash_tree_root()); + attestation + } + + #[test] + fn a_valid_vote_prepares_to_one_pairing_and_resolves_to_accept_either_way() { + let (store, attestation, subnet_id) = signed_vote_across_a_fork_boundary(); + let prepared = prepare_stateful_checks(&store, &attestation, subnet_id); + let sets = prepared.signature_sets(); + assert_eq!(sets.len(), 1); + assert!(bls::verify_signature_sets(&sets)); + assert_eq!(prepared.resolve_batch_passed(), Ok(())); + assert_eq!( + prepare_stateful_checks(&store, &attestation, subnet_id).resolve(), + Ok(()) + ); + } + + /// The signature rule comes before the committee rules, so a vote that + /// breaks both reports the signature, as it did when the signature was + /// verified in place, even though preparation has already run the committee + /// rules by then. + #[test] + fn a_bad_signature_still_outranks_a_later_committee_failure() { + let (store, attestation, subnet_id) = signed_vote_across_a_fork_boundary(); + let mut attestation = with_a_wrong_signature(attestation); + attestation.committee_index = 99; + assert_eq!( + stateful_checks(&store, &attestation, subnet_id), + Outcome::Reject(RejectReason::BadSignature) + ); + // And on the wrong subnet. + let attestation = with_a_wrong_signature(signed_vote_across_a_fork_boundary().1); + assert_eq!( + stateful_checks(&store, &attestation, subnet_id + 1), + Outcome::Reject(RejectReason::BadSignature) + ); + } + + /// A good signature does not hide the committee failure after it. + #[test] + fn a_good_signature_still_reports_the_later_committee_failure() { + let (store, mut attestation, subnet_id) = signed_vote_across_a_fork_boundary(); + attestation.committee_index = 99; + // The signature does not cover `committee_index`, so it still verifies. + let prepared = prepare_stateful_checks(&store, &attestation, subnet_id); + assert_eq!(prepared.signature_sets().len(), 1); + assert_eq!( + prepared.resolve_batch_passed(), + Err(Outcome::Reject(RejectReason::CommitteeIndex)) + ); + } + + /// One batch holding a valid vote and a forged one fails, and resolving + /// each on its own then gives the valid one `Accept` and the forged one + /// its rejection. + #[test] + fn a_failed_batch_resolves_each_vote_on_its_own() { + let (store, good, subnet_id) = signed_vote_across_a_fork_boundary(); + let forged = with_a_wrong_signature(good.clone()); + let good_prepared = prepare_stateful_checks(&store, &good, subnet_id); + let forged_prepared = prepare_stateful_checks(&store, &forged, subnet_id); + let mut sets = good_prepared.signature_sets(); + sets.extend(forged_prepared.signature_sets()); + assert!(!bls::verify_signature_sets(&sets)); + assert_eq!(good_prepared.resolve(), Ok(())); + assert_eq!( + forged_prepared.resolve(), + Err(Outcome::Reject(RejectReason::BadSignature)) + ); + } + + /// A rule that fails before the signature leaves nothing to batch. + #[test] + fn a_vote_for_an_unseen_block_prepares_no_pairing() { + let store = store(0); + let attestation = attestation_at(0, 1, 0); + let prepared = prepare_stateful_checks(&store, &attestation, 0); + assert!(prepared.signature_sets().is_empty()); + assert_eq!( + prepared.resolve_batch_passed(), + Err(Outcome::Ignore(IgnoreReason::UnknownBlock)) + ); + } } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/column.rs b/crates/blockchain/state_transition/src/beacon/gossip/column.rs index e45808eae..3b208eb8d 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/column.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/column.rs @@ -83,9 +83,24 @@ pub fn cheap_checks( /// The rules that need the parent's post-state. Runs on a blocking thread, /// and reads states only from the cache, as [`super::block::stateful_checks`] -/// does. The rules themselves are [`judge_against_parent`]'s, a permit held -/// while they run. +/// does. The rules themselves are [`prepare_against_parent`]'s, a permit held +/// while they run, with the KZG proofs verified on their own. pub fn stateful_checks(store: &Store, sidecar: &DataColumnSidecar) -> Outcome { + match prepare_stateful_checks_fulu(store, sidecar) { + ColumnPrepared::Settled(outcome) => outcome, + prepared @ ColumnPrepared::Kzg { .. } => { + let kzg = { + let _timing = metrics::time_data_column_kzg_verify(); + fork_choice::verify_data_column_sidecar_kzg_proofs(sidecar) + }; + prepared.settle(matches!(kzg, Ok(true))) + } + } +} + +/// [`stateful_checks`] without verifying the KZG proofs; see +/// [`ColumnPrepared`]. +pub fn prepare_stateful_checks_fulu(store: &Store, sidecar: &DataColumnSidecar) -> ColumnPrepared { let header = &sidecar.signed_block_header.message; let mut memo = SignatureMemo::default(); let parent_known = store.has_block(&header.parent_root); @@ -104,9 +119,9 @@ pub fn stateful_checks(store: &Store, sidecar: &DataColumnSidecar) -> Outcome { } else { QueueReason::ParentUnknown }; - return queue_unless_forged(store, sidecar, reason, &mut memo); + return ColumnPrepared::Settled(queue_unless_forged(store, sidecar, reason, &mut memo)); }; - judge_against_parent( + prepare_against_parent( store, sidecar, &parent_state, @@ -448,7 +463,7 @@ pub fn kzg_verdicts(items: &[(&AnyDataColumnSidecar, &PendingKzg)]) -> Vec Outcome { +) -> ColumnPrepared { if let Err(outcome) = judge_before_kzg(store, sidecar, parent_state, past_lookahead, memo) { - return outcome; + return ColumnPrepared::Settled(outcome); } let header = &sidecar.signed_block_header.message; - // [REJECT] Every cell matches its commitment and proof. - let kzg = { - let _timing = metrics::time_data_column_kzg_verify(); - fork_choice::verify_data_column_sidecar_kzg_proofs(sidecar) - }; - if !matches!(kzg, Ok(true)) { - return Outcome::Reject(RejectReason::Kzg); - } // [IGNORE] The sidecar's slot is outside the parent state's proposer // lookahead window (MAY queue). - if past_lookahead == PastLookahead::Queue - && let Err(outcome) = lookahead_covers(parent_state, header.slot) - { - return outcome; + let if_proofs_verify = match past_lookahead { + PastLookahead::Queue => lookahead_covers(parent_state, header.slot) + .err() + .unwrap_or(Outcome::Accept), + PastLookahead::Advance => Outcome::Accept, + }; + ColumnPrepared::Kzg { + pending: PendingKzg { + commitments: sidecar.kzg_commitments.to_vec(), + }, + if_proofs_verify, } - Outcome::Accept } -/// Every rule [`judge_against_parent`] runs before the KZG proofs, in the same +/// A gossip column's stateful rules evaluated up to, but not including, its +/// KZG proofs. See [`prepare_stateful_checks`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ColumnPrepared { + /// A rule before the proofs decided: this is the verdict whatever they say. + Settled(Outcome), + /// Every rule before the proofs passed. + Kzg { + /// What the proofs are checked against. + pending: PendingKzg, + /// The verdict when they verify; [`RejectReason::Kzg`] when they do + /// not. `Accept` unless the lookahead rule (which comes after the + /// proofs) queues the sidecar. + if_proofs_verify: Outcome, + }, +} + +impl ColumnPrepared { + /// What the proofs must be verified against, if this still needs them. + pub fn pending(&self) -> Option<&PendingKzg> { + match self { + Self::Settled(_) => None, + Self::Kzg { pending, .. } => Some(pending), + } + } + + /// The verdict, given whether the proofs verified. Ignored by a + /// [`Self::Settled`] one. + pub fn settle(self, proofs_verify: bool) -> Outcome { + match self { + Self::Settled(outcome) => outcome, + Self::Kzg { + if_proofs_verify, .. + } => { + if proofs_verify { + if_proofs_verify + } else { + Outcome::Reject(RejectReason::Kzg) + } + } + } + } +} + +/// [`stateful_checks`] / [`stateful_checks_gloas`] without verifying the KZG +/// proofs, for either fork's sidecar. +pub fn prepare_stateful_checks(store: &Store, sidecar: &AnyDataColumnSidecar) -> ColumnPrepared { + match sidecar { + AnyDataColumnSidecar::Fulu(sidecar) => prepare_stateful_checks_fulu(store, sidecar), + AnyDataColumnSidecar::Gloas(sidecar) => prepare_stateful_checks_gloas(store, sidecar), + } +} + +/// Whether one batched `verify_cell_kzg_proof_batch` call over all of `items` +/// verifies. `false` does not say which sidecar is at fault: the caller +/// verifies each with [`kzg_verdict`]'s rule ([`kzg_proofs_verify`]) after. +pub fn kzg_batch_verifies(items: &[(&AnyDataColumnSidecar, &PendingKzg)]) -> bool { + let refs: Vec> = items.iter().map(|(sidecar, _)| (*sidecar).into()).collect(); + let columns: Vec> = refs + .iter() + .zip(items) + .map(|(sidecar, (_, pending))| sidecar.cells(&pending.commitments)) + .collect(); + let batch = { + let _timing = metrics::time_data_column_kzg_verify(); + fork_choice::verify_column_cells_batch(&columns) + }; + matches!(batch, Ok(true)) +} + +/// Whether one sidecar's proofs verify on their own. +pub fn kzg_proofs_verify(sidecar: &AnyDataColumnSidecar, pending: &PendingKzg) -> bool { + matches!(kzg_verdict(sidecar, pending), ChainVerdict::Keep) +} + +/// Every rule [`prepare_against_parent`] runs before the KZG proofs, in the same /// order, so `Ok` means "accepted unless the proofs fail" and an `Err` is the /// verdict whatever the proofs say. /// @@ -684,19 +780,41 @@ pub fn cheap_checks_gloas( /// is queued too where the specification rejects it, the same deviation /// fulu's parent rule makes until a bad-block cache exists. pub fn stateful_checks_gloas(store: &Store, sidecar: &gloas::DataColumnSidecar) -> Outcome { - let kzg_commitments = match gloas_before_kzg(store, sidecar) { - Ok(commitments) => commitments, - Err(outcome) => return outcome, - }; - // [REJECT] The sidecar's column data passes KZG verification. - let kzg = { - let _timing = metrics::time_data_column_kzg_verify(); - fork_choice::gloas_verify_data_column_sidecar_kzg_proofs(sidecar, &kzg_commitments) - }; - if !matches!(kzg, Ok(true)) { - return Outcome::Reject(RejectReason::Kzg); + match prepare_stateful_checks_gloas(store, sidecar) { + ColumnPrepared::Settled(outcome) => outcome, + prepared @ ColumnPrepared::Kzg { .. } => { + // [REJECT] The sidecar's column data passes KZG verification. + let proofs_verify = { + let _timing = metrics::time_data_column_kzg_verify(); + let commitments = prepared.pending().map(|pending| &pending.commitments[..]); + commitments.is_some_and(|commitments| { + matches!( + fork_choice::gloas_verify_data_column_sidecar_kzg_proofs( + sidecar, + commitments + ), + Ok(true) + ) + }) + }; + prepared.settle(proofs_verify) + } + } +} + +/// [`stateful_checks_gloas`] without verifying the KZG proofs; see +/// [`ColumnPrepared`]. +pub fn prepare_stateful_checks_gloas( + store: &Store, + sidecar: &gloas::DataColumnSidecar, +) -> ColumnPrepared { + match gloas_before_kzg(store, sidecar) { + Ok(commitments) => ColumnPrepared::Kzg { + pending: PendingKzg { commitments }, + if_proofs_verify: Outcome::Accept, + }, + Err(outcome) => ColumnPrepared::Settled(outcome), } - Outcome::Accept } /// Every rule [`stateful_checks_gloas`] runs before the KZG proofs, in the @@ -1851,6 +1969,51 @@ mod tests { ); } + #[test] + fn the_batch_check_passes_for_valid_columns_and_fails_with_one_forged() { + let mut items = real_sidecars(2, &[0, 7]); + let refs = |items: &[(AnyDataColumnSidecar, PendingKzg)]| { + let pairs: Vec<_> = items.iter().map(|(s, p)| (s, p)).collect(); + kzg_batch_verifies(&pairs) + }; + assert!(refs(&items)); + let AnyDataColumnSidecar::Fulu(forged) = &mut items[0].0 else { + unreachable!("real_sidecars builds fulu sidecars"); + }; + let mut cell = forged.column[0].to_vec(); + cell[0] ^= 1; + forged.column[0] = libssz_types::SszVector::try_from(cell).expect("exact cell size"); + assert!(!refs(&items)); + assert!(!kzg_proofs_verify(&items[0].0, &items[0].1)); + assert!(kzg_proofs_verify(&items[1].0, &items[1].1)); + } + + #[test] + fn a_prepared_column_settles_on_its_proofs_unless_a_rule_before_them_decided() { + let queued = Outcome::Queue(QueueReason::ParentNotReady); + let kzg = ColumnPrepared::Kzg { + pending: PendingKzg { + commitments: Vec::new(), + }, + if_proofs_verify: queued, + }; + assert!(kzg.pending().is_some()); + // The lookahead rule comes after the proofs: forged proofs are + // rejected even where the lookahead would have queued the sidecar. + assert_eq!(kzg.clone().settle(true), queued); + assert_eq!(kzg.settle(false), Outcome::Reject(RejectReason::Kzg)); + let settled = ColumnPrepared::Settled(Outcome::Reject(RejectReason::WrongProposer)); + assert!(settled.pending().is_none()); + assert_eq!( + settled.clone().settle(true), + Outcome::Reject(RejectReason::WrongProposer) + ); + assert_eq!( + settled.settle(false), + Outcome::Reject(RejectReason::WrongProposer) + ); + } + /// Prints the time of one batch over `k` columns against `k` separate /// calls. Run with `--ignored --nocapture`. #[test] diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 1d5019069..c9a6bcc70 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -32,6 +32,7 @@ use std::num::NonZeroUsize; use lru::LruCache; +use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants::MAXIMUM_GOSSIP_CLOCK_DISPARITY; use crate::beacon::containers::{AttestationData, BeaconState, gloas}; @@ -44,6 +45,103 @@ use crate::beacon::precheck::PrecheckError; use crate::beacon::preset; use crate::beacon::primitives::{Epoch, Root, Slot, ValidatorIndex}; +/// A signature rule whose pairing the caller evaluates, alone or batched with +/// other messages' (see [`Prepared`]). +pub struct SignatureStep { + /// `None` when the key or the signature failed validation while the set + /// was built, which fails exactly where [`bls::verify`] returning false + /// does. + pub set: Option, + /// The verdict when the signature does not verify. + pub on_failure: Outcome, +} + +/// A stateful rule evaluated up to, but not including, its signature pairings. +/// +/// The rules run as a sequence of signature checks and non-signature rules, +/// and preparation stops at the first non-signature rule that fails (later +/// rules may depend on it). So the original order is always some signature +/// steps, in order, then one end: the failed rule, or the payload `Accept` +/// carries. Every signature is a deferred step rather than a verified one, so +/// a caller can fold the pairings of many messages into one +/// [`bls::verify_signature_sets`] call. The non-signature rules are pure +/// reads, so running them before the pairings and then taking the first +/// failure in the original order gives the verdict the one-by-one order did: +/// a message with a bad signature and a bad committee index is still a +/// `BadSignature`, for instance, because the signature step sits before the +/// end. +pub struct Prepared { + signatures: Vec, + end: Option>, +} + +impl Default for Prepared { + fn default() -> Self { + Self { + signatures: Vec::new(), + end: None, + } + } +} + +impl Prepared { + /// Add a signature rule after the steps so far. + pub(crate) fn signature(&mut self, set: Option, on_failure: Outcome) { + self.signatures.push(SignatureStep { set, on_failure }); + } + + /// End with a failed non-signature rule. + pub(crate) fn fail(mut self, outcome: Outcome) -> Self { + self.end = Some(Err(outcome)); + self + } + + /// End with every non-signature rule passed, carrying what `Accept` hands + /// on. + pub(crate) fn accept(mut self, payload: T) -> Self { + self.end = Some(Ok(payload)); + self + } + + /// The pairings a batch must verify for this message's verdict to be + /// settled: the signature sets up to the first one that could not be built, + /// which fails whatever the batch says. Sets after that point are never + /// reached, so batching them would only let one bad message sink a batch + /// for nothing. + pub fn signature_sets(&self) -> Vec { + self.signatures.iter().map_while(|step| step.set).collect() + } + + /// Walk the steps in the original order: the first failure is the verdict, + /// else `Ok` with the payload. A set that could not be built fails without + /// consulting `verifies`. + fn resolve_with( + self, + mut verifies: impl FnMut(&bls::SignatureSet) -> bool, + ) -> Result { + for SignatureStep { set, on_failure } in self.signatures { + if !set.as_ref().is_some_and(&mut verifies) { + return Err(on_failure); + } + } + self.end + .unwrap_or(Err(Outcome::Ignore(IgnoreReason::Internal))) + } + + /// Resolve by evaluating each pairing on its own, only as far as the walk + /// reaches. What the one-by-one callers (the Beacon API, the spec vectors) + /// and a failed batch's fallback use. + pub fn resolve(self) -> Result { + self.resolve_with(bls::SignatureSet::verify) + } + + /// Resolve knowing a [`bls::verify_signature_sets`] call over + /// [`Self::signature_sets`] passed: every set that exists verifies. + pub fn resolve_batch_passed(self) -> Result { + self.resolve_with(|_| true) + } +} + /// A gossip message's verdict. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Outcome { @@ -1101,3 +1199,81 @@ mod tests { assert_eq!(ancestor_at(&state, at_block_root, 0), None); } } + +#[cfg(test)] +mod prepared_tests { + use super::*; + use crate::beacon::helpers::test_state::{secret_key_for, sign_for}; + use crate::beacon::primitives::BlsPubkey; + + const FAILED_SIGNATURE: Outcome = Outcome::Reject(RejectReason::BadSignature); + + fn set_for(signer: usize, message: Root) -> Option { + let pubkey = BlsPubkey(secret_key_for(signer).sk_to_pk().to_bytes()); + bls::SignatureSet::single(&pubkey, message, &sign_for(signer, message)) + } + + fn forged_set(message: Root) -> Option { + let pubkey = BlsPubkey(secret_key_for(1).sk_to_pk().to_bytes()); + bls::SignatureSet::single(&pubkey, message, &sign_for(2, message)) + } + + #[test] + fn a_prepared_check_with_every_step_passing_accepts_with_its_payload() { + let mut prepared = Prepared::default(); + prepared.signature(set_for(1, Root::repeat_byte(1)), FAILED_SIGNATURE); + let prepared = prepared.accept(7u8); + assert_eq!(prepared.signature_sets().len(), 1); + assert_eq!(prepared.resolve(), Ok(7)); + } + + #[test] + fn the_first_failing_step_in_the_original_order_is_the_verdict() { + // Signature, then a failed rule: a good signature lets the rule speak, + // a bad one speaks first. + let rule = Outcome::Reject(RejectReason::CommitteeIndex); + let build = |set| { + let mut prepared = Prepared::<()>::default(); + prepared.signature(set, FAILED_SIGNATURE); + prepared.fail(rule) + }; + assert_eq!( + build(set_for(1, Root::repeat_byte(1))).resolve_batch_passed(), + Err(rule) + ); + assert_eq!( + build(forged_set(Root::repeat_byte(1))).resolve(), + Err(FAILED_SIGNATURE) + ); + } + + #[test] + fn a_set_that_could_not_be_built_fails_like_a_failed_pairing() { + let mut prepared = Prepared::<()>::default(); + prepared.signature(None, FAILED_SIGNATURE); + let prepared = prepared.accept(()); + // Nothing to batch, and the batch passing does not rescue it. + assert!(prepared.signature_sets().is_empty()); + assert_eq!(prepared.resolve_batch_passed(), Err(FAILED_SIGNATURE)); + } + + #[test] + fn only_the_sets_a_verdict_can_depend_on_are_batched() { + let mut prepared = Prepared::<()>::default(); + prepared.signature(set_for(1, Root::repeat_byte(1)), FAILED_SIGNATURE); + prepared.signature(None, FAILED_SIGNATURE); + prepared.signature(set_for(2, Root::repeat_byte(2)), FAILED_SIGNATURE); + // The second step fails whatever the batch says, so the third is never + // reached and must not be able to sink a batch. + assert_eq!(prepared.signature_sets().len(), 1); + } + + #[test] + fn a_prepared_check_with_no_verdict_is_internal() { + let prepared = Prepared::<()>::default(); + assert_eq!( + prepared.resolve(), + Err(Outcome::Ignore(IgnoreReason::Internal)) + ); + } +} diff --git a/crates/blockchain/state_transition/src/beacon/gossip/sync_committee.rs b/crates/blockchain/state_transition/src/beacon/gossip/sync_committee.rs index 919341bb4..96a4e95da 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/sync_committee.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/sync_committee.rs @@ -23,7 +23,7 @@ use std::num::NonZeroUsize; use ethlambda_storage::CacheKey; use lru::LruCache; -use super::{IgnoreReason, Outcome, RejectReason, is_current_slot}; +use super::{IgnoreReason, Outcome, Prepared, RejectReason, is_current_slot}; use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants::SYNC_COMMITTEE_SUBNET_COUNT; @@ -200,8 +200,20 @@ pub fn message_stateful_checks( message: &SyncCommitteeMessage, subnet_id: u64, ) -> Result, Outcome> { - let state = head_state(store)?; - check_message(&state, &store.config(), message, Some(subnet_id)) + prepare_message_stateful_checks(store, message, subnet_id).resolve() +} + +/// [`message_stateful_checks`] without evaluating the signature: see +/// [`prepare_message`] and [`Prepared`]. +pub fn prepare_message_stateful_checks( + store: &Store, + message: &SyncCommitteeMessage, + subnet_id: u64, +) -> Prepared> { + match head_state(store) { + Ok(state) => prepare_message(&state, &store.config(), message, Some(subnet_id)), + Err(outcome) => Prepared::default().fail(outcome), + } } /// The state-dependent rules for one message. With `subnet_id`, the seats kept @@ -213,14 +225,27 @@ pub fn check_message( message: &SyncCommitteeMessage, subnet_id: Option, ) -> Result, Outcome> { + prepare_message(state, config, message, subnet_id).resolve() +} + +/// [`check_message`] without evaluating the signature. The signature is the +/// last rule, so the steps are either a seat rule that failed early, or the +/// one signature step and the seats it covers. +pub fn prepare_message( + state: &BeaconState, + config: &Config, + message: &SyncCommitteeMessage, + subnet_id: Option, +) -> Prepared> { + let mut prepared = Prepared::default(); // [REJECT] The validator index is valid. let Ok(validator) = state.validator(message.validator_index) else { - return Err(Outcome::Reject(RejectReason::UnknownValidator)); + return prepared.fail(Outcome::Reject(RejectReason::UnknownValidator)); }; // A period the head state's committees cannot answer for is not the // sender's fault. let Ok(committee) = sync_committee_for_slot(state, message.slot) else { - return Err(Outcome::Ignore(IgnoreReason::SyncCommitteeUnavailable)); + return prepared.fail(Outcome::Ignore(IgnoreReason::SyncCommitteeUnavailable)); }; let mut seats = sync_committee_seats(committee, &validator.pubkey); match subnet_id { @@ -228,13 +253,13 @@ pub fn check_message( seats.retain(|&(seat_subnet, _)| seat_subnet == subnet); // [REJECT] The validator is in the subcommittee of this subnet. if seats.is_empty() { - return Err(Outcome::Reject(RejectReason::WrongSubnet)); + return prepared.fail(Outcome::Reject(RejectReason::WrongSubnet)); } } None => { // [REJECT] The validator is in the sync committee. if seats.is_empty() { - return Err(Outcome::Reject(RejectReason::NotInCommittee)); + return prepared.fail(Outcome::Reject(RejectReason::NotInCommittee)); } } } @@ -249,10 +274,11 @@ pub fn check_message( index: message.validator_index, pubkey: validator.pubkey, }; - if !bls::verify(&validator, signing_root, &message.signature) { - return Err(Outcome::Reject(RejectReason::BadSignature)); - } - Ok(seats) + prepared.signature( + bls::SignatureSet::single(&validator, signing_root, &message.signature), + Outcome::Reject(RejectReason::BadSignature), + ); + prepared.accept(seats) } /// The rules for a message submitted through the Beacon API: the clock, then @@ -573,6 +599,59 @@ mod tests { ); } + // --- prepared messages ------------------------------------------------ + + #[test] + fn a_valid_message_prepares_one_pairing_and_resolves_to_its_seats_either_way() { + let (store, _) = store_with_head(); + let message = message_from(1, 1); + let prepared = prepare_message_stateful_checks(&store, &message, subnet_of(1)); + let sets = prepared.signature_sets(); + assert_eq!(sets.len(), 1); + assert!(bls::verify_signature_sets(&sets)); + let seats = vec![(subnet_of(1), 1 % SYNC_SUBCOMMITTEE_SIZE)]; + assert_eq!(prepared.resolve_batch_passed(), Ok(seats.clone())); + assert_eq!( + prepare_message_stateful_checks(&store, &message, subnet_of(1)).resolve(), + Ok(seats) + ); + } + + /// The seat rules come before the signature, so a message that breaks one + /// and carries a bad signature reports the seat rule, and prepares no + /// pairing for a batch to verify. + #[test] + fn a_wrong_subnet_outranks_a_bad_signature() { + let (store, _) = store_with_head(); + let forged = message_from(1, 2); + let wrong_subnet = (subnet_of(1) + 1) % 4; + let prepared = prepare_message_stateful_checks(&store, &forged, wrong_subnet); + assert!(prepared.signature_sets().is_empty()); + assert_eq!( + prepared.resolve(), + Err(Outcome::Reject(RejectReason::WrongSubnet)) + ); + assert_eq!( + message_stateful_checks(&store, &forged, wrong_subnet), + Err(Outcome::Reject(RejectReason::WrongSubnet)) + ); + } + + #[test] + fn a_failed_batch_resolves_each_message_on_its_own() { + let (store, _) = store_with_head(); + let good = prepare_message_stateful_checks(&store, &message_from(1, 1), subnet_of(1)); + let forged = prepare_message_stateful_checks(&store, &message_from(1, 2), subnet_of(1)); + let mut sets = good.signature_sets(); + sets.extend(forged.signature_sets()); + assert!(!bls::verify_signature_sets(&sets)); + assert!(good.resolve().is_ok()); + assert_eq!( + forged.resolve(), + Err(Outcome::Reject(RejectReason::BadSignature)) + ); + } + // --- contributions ---------------------------------------------------- /// Searches for an aggregator among the subcommittee's first members: a diff --git a/crates/common/crypto/src/bls.rs b/crates/common/crypto/src/bls.rs index ae57ef6d0..bf9e74b7f 100644 --- a/crates/common/crypto/src/bls.rs +++ b/crates/common/crypto/src/bls.rs @@ -75,6 +75,21 @@ //! A key that fails is never cached in either structure, so an invalid key //! costs a full check every time, exactly as before, and cannot grow either. //! +//! # Why a message is hashed once +//! +//! Hashing a signing root to a G2 point costs about as much as a hundred +//! microseconds of pairing work, and the same root is signed by every attester +//! of a slot. [`verify`], [`aggregate_verify`], [`fast_aggregate_verify`] and +//! [`SignatureSet`] therefore do not call `blst`'s one-shot verifiers, which +//! hash on every call: they take the hash from a small two-generation cache +//! (`hashed_messages`) and run the pairing equation over it themselves (in +//! `primitives`, the only module with raw `blst` point calls). Decoding and +//! validation are unchanged, and `primitives::pairing_check` refuses an +//! identity key explicitly, since a hand-built pairing would otherwise skip it +//! where `blst`'s verifier fails it. [`verify_signature_sets`] additionally +//! groups its sets by message, so a batch pays one pairing per distinct +//! message. +//! //! # `bool` versus `Result` //! //! The verification functions ([`verify`], [`aggregate_verify`], @@ -125,8 +140,8 @@ use std::collections::HashMap; use std::sync::{LazyLock, OnceLock, RwLock}; +use blst::MultiPoint; use blst::min_pk::{AggregatePublicKey, AggregateSignature, PublicKey, Signature}; -use blst::{BLST_ERROR, MultiPoint}; use rayon::prelude::*; use ethlambda_types::beacon::error::{Error, Result}; @@ -137,6 +152,9 @@ use ethlambda_types::beacon::primitives::{ mod batch; pub use batch::{SignatureSet, verify_signature_sets}; +mod hashed_messages; +mod primitives; + /// The ciphersuite the specification pins BLS signatures to: the IETF BLS /// draft's proof-of-possession scheme over BLS12-381's G2, using SHA-256 in /// the XMD hash-to-curve construction. @@ -546,8 +564,19 @@ pub fn verify(pubkey: &K, message: Root, signature: &BlsSignatu let Ok(signature) = Signature::sig_validate(signature.as_ref(), false) else { return false; }; - let result = signature.verify(false, message.as_slice(), DST, &[], &pubkey, false); - result == BLST_ERROR::BLST_SUCCESS + pairing_verify(&pubkey, &message, &signature) +} + +/// `e(pubkey, H(message)) == e(G1, signature)` for already validated points, +/// with `H(message)` from [`hashed_messages`]'s cache. +/// +/// The one pairing every single check goes through. An identity `pubkey` (a +/// sum of cancelling keys) is refused by [`primitives::pairing_check`]. +fn pairing_verify(pubkey: &PublicKey, message: &Root, signature: &Signature) -> bool { + let hashed = hashed_messages::hashed_message(message); + let key = blst::blst_p1_affine::from(*pubkey); + let signature = blst::blst_p2_affine::from(*signature); + primitives::pairing_check(&[(&key, &hashed)], &signature) } /// The specification's `bls.Aggregate(signatures) -> BLSSignature`. @@ -596,10 +625,17 @@ pub fn aggregate_verify( let Some(points) = validated_pubkeys(pubkeys) else { return false; }; - let point_refs: Vec<&PublicKey> = points.iter().collect(); - let message_refs: Vec<&[u8]> = messages.iter().map(Root::as_slice).collect(); - let result = signature.aggregate_verify(false, &message_refs, DST, &point_refs, false); - result == BLST_ERROR::BLST_SUCCESS + // Hashing the distinct messages is the bulk of this check, and the cache + // is thread-safe, so the hashes (cached or not) fan out like blst's did. + let hashed: Vec = messages + .par_iter() + .with_min_len(KEY_VALIDATE_MIN_PAR_LEN) + .map(hashed_messages::hashed_message) + .collect(); + let keys: Vec = points.iter().copied().map(Into::into).collect(); + let pairs: Vec<(&blst::blst_p1_affine, &blst::blst_p2_affine)> = + keys.iter().zip(&hashed).collect(); + primitives::pairing_check(&pairs, &signature.into()) } /// The specification's @@ -632,9 +668,7 @@ pub fn fast_aggregate_verify( let Some(aggregated) = sum_validated_points(&points) else { return false; }; - let result = - signature.fast_aggregate_verify_pre_aggregated(false, message.as_slice(), DST, &aggregated); - result == BLST_ERROR::BLST_SUCCESS + pairing_verify(&aggregated, &message, &signature) } /// `specs/altair/bls.md`'s `eth_aggregate_pubkeys(pubkeys) -> BLSPubkey`. @@ -694,11 +728,54 @@ pub fn key_validate(pubkey: &K) -> bool { mod committee_sums; pub use committee_sums::{AggregatePubkey, CommitteeKeySums}; +mod warm; +pub use warm::warm_pubkeys; + #[cfg(test)] mod tests { use super::*; use blst::min_pk::AggregatePublicKey; + #[test] + fn aggregate_verify_agrees_with_blst_over_distinct_and_repeated_messages() { + // 40 signers is past the parallel hashing threshold; messages repeat + // in the second round. + for distinct in [40u8, 3] { + let keys: Vec = (0..40u64) + .map(|seed| { + let mut ikm = [0x3cu8; 32]; + ikm[..8].copy_from_slice(&seed.to_le_bytes()); + blst::min_pk::SecretKey::key_gen(&ikm, &[]).unwrap() + }) + .collect(); + let pubkeys: Vec = keys + .iter() + .map(|key| BlsPubkey(key.sk_to_pk().to_bytes())) + .collect(); + let messages: Vec = (0..40u8).map(|i| Root::from([i % distinct; 32])).collect(); + let signatures: Vec = keys + .iter() + .zip(&messages) + .map(|(key, message)| { + BlsSignature(key.sign(message.as_slice(), DST, &[]).to_bytes()) + }) + .collect(); + let aggregated = aggregate(&signatures).unwrap(); + assert!(aggregate_verify(&pubkeys, &messages, &aggregated)); + let mut swapped = messages.clone(); + swapped.swap(0, 1); + assert_eq!( + aggregate_verify(&pubkeys, &swapped, &aggregated), + messages[0] == messages[1] + ); + assert!(!aggregate_verify( + &pubkeys[1..], + &messages[1..], + &aggregated + )); + } + } + #[test] fn key_validate_rejects_the_all_zero_pubkey() { assert!(!key_validate(&BlsPubkey::default())); diff --git a/crates/common/crypto/src/bls/batch.rs b/crates/common/crypto/src/bls/batch.rs index adda00f62..3d79e77f0 100644 --- a/crates/common/crypto/src/bls/batch.rs +++ b/crates/common/crypto/src/bls/batch.rs @@ -11,6 +11,18 @@ //! them, so a block with a dozen signature sets spends a fraction of what a //! dozen separate checks do. //! +//! # Why sets are grouped by message +//! +//! Validators attesting to the same data sign the same message, and bilinearity +//! lets their equations share a pairing: for the sets `i` of one message `m`, +//! `prod_i e(r_i * pk_i, H(m)) = e(sum_i r_i * pk_i, H(m))`. The batch sorts the +//! sets by message, sums `r_i * pk_i` per run of equal messages with one +//! multi-scalar multiplication, and checks +//! `prod_g e(sum_g r_i * pk_i, H(m_g)) == e(g1, sum_i r_i * sig_i)`: one Miller +//! loop per *distinct* message, one multiplication for all the signatures, one +//! final exponentiation. The grouping also means each distinct message is +//! hashed once, through the cache in `hashed_messages`. +//! //! # Why the weights are random //! //! Multiplying the equations together unweighted is unsound: an attacker who @@ -26,6 +38,12 @@ //! operating system's entropy and reseeded periodically, and a weight of zero //! is redrawn, since it would drop its set from the equation altogether. //! +//! Grouping adds two refusals. A weighted signature sum that is the point at +//! infinity fails the batch: the right-hand side would be trivial, and a pair +//! of keys `P` and `-P` whose weighted sum also vanished would let the +//! left-hand side drop out too. And a group whose weighted key sum is the +//! identity fails, as the pairing check refuses an identity key. +//! //! # What a batch answers //! //! Only whether every set holds. When it fails it cannot say which set did; @@ -34,21 +52,19 @@ //! [`super::verify`] or [`super::fast_aggregate_verify`] gives for the same //! inputs. -use blst::BLST_ERROR; -use blst::blst_scalar; +use std::ops::Range; + use blst::min_pk::{PublicKey, Signature}; +use blst::{blst_p1_affine, blst_p2_affine}; use rand::RngExt; +use rayon::prelude::*; use ethlambda_types::beacon::primitives::{BlsSignature, Root}; -use super::{AggregatePubkey, DST, PubkeySource, validated_pubkey, validated_pubkeys}; - -/// How many bits of each random weight the batch check uses. -const RAND_BITS: usize = 64; - -/// The bit of a compressed G1 point's first byte that marks the point at -/// infinity. -const INFINITY_FLAG: u8 = 0b0100_0000; +use super::hashed_messages::hashed_message; +use super::{ + AggregatePubkey, PubkeySource, pairing_verify, primitives, validated_pubkey, validated_pubkeys, +}; /// One signature check, resolved and ready, that may be verified alone or in a /// batch. @@ -63,9 +79,6 @@ pub struct SignatureSet { pubkey: PublicKey, message: Root, signature: Signature, - /// Whether `pubkey` is a sum of keys, checked the way - /// `bls.FastAggregateVerify` checks one, rather than a single registry key. - aggregated: bool, } impl SignatureSet { @@ -84,7 +97,6 @@ impl SignatureSet { pubkey, message, signature, - aggregated: false, }) } @@ -110,7 +122,6 @@ impl SignatureSet { pubkey, message, signature, - aggregated: true, }) } @@ -131,67 +142,61 @@ impl SignatureSet { pubkey: key.to_public_key(), message, signature, - aggregated: true, }) } /// The pairing check on its own: the same answer [`super::verify`] or /// [`super::fast_aggregate_verify`] gives for these inputs. pub fn verify(&self) -> bool { - let result = if self.aggregated { - self.signature.fast_aggregate_verify_pre_aggregated( - false, - self.message.as_slice(), - DST, - &self.pubkey, - ) - } else { - self.signature.verify( - false, - self.message.as_slice(), - DST, - &[], - &self.pubkey, - false, - ) - }; - result == BLST_ERROR::BLST_SUCCESS + pairing_verify(&self.pubkey, &self.message, &self.signature) } /// Whether the key is the identity, which no signature can validly be /// made under. Only a sum of keys can be: a single registry key is /// rejected by `key_validate` if it is. /// - /// blst rejects the same input itself (`BLST_PK_IS_INFINITY`, from the - /// pairing aggregation both the single and the batch check go through), - /// so this makes the existing rule explicit and changes no behavior. + /// The pairing check refuses the same input itself, so this makes the + /// rule explicit where a reader of the batch looks for it, and lets the + /// batch return before doing any group arithmetic. fn key_is_identity(&self) -> bool { - self.pubkey.to_bytes()[0] & INFINITY_FLAG != 0 + primitives::key_is_identity(&self.pubkey) } } -/// A random weight in `1..2^64`, as the little-endian scalar `blst` expects. -fn random_weight(rng: &mut impl RngExt) -> blst_scalar { - let weight = loop { +/// A random weight in `1..2^64`. +fn random_weight(rng: &mut impl RngExt) -> u64 { + loop { let candidate: u64 = rng.random(); if candidate != 0 { break candidate; } - }; - let mut scalar = blst_scalar::default(); - scalar.b[..8].copy_from_slice(&weight.to_le_bytes()); - scalar + } } -/// Whether every set verifies, in one random-linear-combination check: -/// `verify_multiple_aggregate_signatures` with 64-bit scalars from the -/// OS-seeded RNG and one final exponentiation. An empty slice verifies. +/// The runs of equal messages in `sets` taken in `order`, as index ranges into +/// `order`. `order` must already be sorted by message. +fn message_runs(sets: &[SignatureSet], order: &[usize]) -> Vec> { + let mut runs = Vec::new(); + let mut start = 0; + for end in 1..=order.len() { + if end == order.len() || sets[order[end]].message != sets[order[start]].message { + runs.push(start..end); + start = end; + } + } + runs +} + +/// Whether every set verifies, in one random-linear-combination check grouped +/// by message: one pairing per distinct message and one final exponentiation. +/// An empty slice verifies. /// -/// See the module documentation for why the weights are random. A single set -/// skips them and is verified directly, since one equation needs no -/// combining. A set whose key is the identity fails the batch, as it fails -/// [`SignatureSet::verify`]. blst rejects that input itself, so the explicit -/// check only states the rule where a reader of the batch looks for it. +/// See the module documentation for the equation and why the weights are +/// random (64-bit, nonzero, from the OS-seeded RNG). A single set skips them +/// and is verified directly, since one equation needs no combining. A set +/// whose key is the identity fails the batch, as it fails +/// [`SignatureSet::verify`], and so does a batch whose weighted signature sum +/// is the point at infinity. pub fn verify_signature_sets(sets: &[SignatureSet]) -> bool { match sets { [] => return true, @@ -201,23 +206,36 @@ pub fn verify_signature_sets(sets: &[SignatureSet]) -> bool { if sets.iter().any(SignatureSet::key_is_identity) { return false; } + let mut order: Vec = (0..sets.len()).collect(); + order.sort_unstable_by(|&a, &b| sets[a].message.as_slice().cmp(sets[b].message.as_slice())); + let mut rng = rand::rng(); - let weights: Vec = sets.iter().map(|_| random_weight(&mut rng)).collect(); - let messages: Vec<&[u8]> = sets.iter().map(|set| set.message.as_slice()).collect(); - let pubkeys: Vec<&PublicKey> = sets.iter().map(|set| &set.pubkey).collect(); - let signatures: Vec<&Signature> = sets.iter().map(|set| &set.signature).collect(); - // Keys and signatures were validated when the sets were built. - let result = Signature::verify_multiple_aggregate_signatures( - &messages, - DST, - &pubkeys, - false, - &signatures, - false, - &weights, - RAND_BITS, - ); - result == BLST_ERROR::BLST_SUCCESS + let weights: Vec = order.iter().map(|_| random_weight(&mut rng)).collect(); + let keys: Vec = order.iter().map(|&i| sets[i].pubkey).collect(); + let signatures: Vec = order.iter().map(|&i| sets[i].signature).collect(); + + // If the weighted signatures summed to the identity the right-hand side of + // the equation would be trivial. + let signature_sum = primitives::weighted_signature_sum(&signatures, &weights); + if primitives::signature_is_identity(&signature_sum) { + return false; + } + + let runs = message_runs(sets, &order); + let groups: Vec<(PublicKey, blst_p2_affine)> = runs + .par_iter() + .map(|run| { + let key = primitives::weighted_key_sum(&keys[run.clone()], &weights[run.clone()]); + (key, hashed_message(&sets[order[run.start]].message)) + }) + .collect(); + let group_keys: Vec = groups.iter().map(|(key, _)| (*key).into()).collect(); + let pairs: Vec<(&blst_p1_affine, &blst_p2_affine)> = group_keys + .iter() + .zip(&groups) + .map(|(key, (_, hashed))| (key, hashed)) + .collect(); + primitives::pairing_check(&pairs, &signature_sum.into()) } #[cfg(test)] @@ -225,7 +243,7 @@ mod tests { use blst::min_pk::SecretKey; use super::*; - use crate::bls; + use crate::bls::{self, DST}; use ethlambda_types::beacon::primitives::BlsPubkey; fn secret_key(seed: u8) -> SecretKey { @@ -431,8 +449,110 @@ mod tests { let mut rng = rand::rng(); for _ in 0..10_000 { let weight = random_weight(&mut rng); - assert!(weight.b[..8].iter().any(|byte| *byte != 0)); - assert!(weight.b[8..].iter().all(|byte| *byte == 0)); + assert_ne!(weight, 0); + } + } + + /// `sets` of `count` single-key sets signing `msg`, valid unless `bad` is + /// given, whose index then signs a different message. + fn sets_over(msg: Root, seeds: std::ops::Range, bad: Option) -> Vec { + seeds + .map(|i| { + let key = secret_key(i); + let signed = if Some(i) == bad { message(250) } else { msg }; + SignatureSet::single(&pubkey_of(&key), msg, &sign(&key, signed)).unwrap() + }) + .collect() + } + + #[test] + fn a_bad_set_among_sets_sharing_its_message_fails_the_batch() { + for bad in [30u8, 33, 37] { + let sets = sets_over(message(77), 30..38, Some(bad)); + assert!(!verify_signature_sets(&sets), "bad set {bad}"); + // Another message group alongside must not mask it. + let mut mixed = valid_sets(3); + mixed.extend(sets); + assert!(!verify_signature_sets(&mixed), "bad set {bad} in a mix"); + } + assert!(verify_signature_sets(&sets_over(message(77), 30..38, None))); + } + + #[test] + fn cancelling_keys_in_a_multi_set_group_never_verify() { + let key = secret_key(40); + let pubkey = pubkey_of(&key); + let msg = message(40); + let infinity = BlsSignature(bls::G2_POINT_AT_INFINITY); + let real = sign(&key, msg); + for (first, second) in [(infinity, infinity), (real, real), (real, infinity)] { + let plus = SignatureSet::single(&pubkey, msg, &first).unwrap(); + let minus = SignatureSet::single(&negated(pubkey), msg, &second).unwrap(); + for _ in 0..20 { + let mut sets = valid_sets(2); + sets.push(plus); + sets.push(minus); + assert!(!verify_signature_sets(&sets)); + assert!(!verify_signature_sets(&[plus, minus])); + } + } + } + + #[test] + fn a_weighted_signature_sum_at_infinity_is_refused() { + // Both signatures are the identity, so their weighted sum always is. + let infinity = BlsSignature(bls::G2_POINT_AT_INFINITY); + let sets: Vec = (50..54) + .map(|i| { + SignatureSet::single(&pubkey_of(&secret_key(i)), message(50), &infinity).unwrap() + }) + .collect(); + for _ in 0..10 { + assert!(!verify_signature_sets(&sets)); + } + let signatures: Vec = sets.iter().map(|set| set.signature).collect(); + let sum = primitives::weighted_signature_sum(&signatures, &[3, 5, 7, 9]); + assert!(primitives::signature_is_identity(&sum)); + } + + /// The verdict plain blst gives for every set alone, ANDed. + fn blst_checks_each(sets: &[SignatureSet]) -> bool { + sets.iter().all(|set| { + set.signature + .verify(false, set.message.as_slice(), DST, &[], &set.pubkey, false) + == blst::BLST_ERROR::BLST_SUCCESS + }) + } + + #[test] + fn the_grouped_batch_agrees_with_blst_checking_each_set() { + use rand::RngExt; + let mut rng = rand::rng(); + for size in 2..=20usize { + for round in 0..4 { + // Few distinct messages so groups are shared, a few keys so + // the same key signs several messages. + let distinct: u8 = rng.random_range(1..=size.min(6) as u8); + let corrupt = round % 2 == 1; + let bad_at = rng.random_range(0..size); + let sets: Vec = (0..size) + .map(|i| { + let key = secret_key(rng.random_range(60..70)); + let msg = message(rng.random_range(0..distinct)); + let signed = if corrupt && i == bad_at { + message(200) + } else { + msg + }; + SignatureSet::single(&pubkey_of(&key), msg, &sign(&key, signed)).unwrap() + }) + .collect(); + assert_eq!( + verify_signature_sets(&sets), + blst_checks_each(&sets), + "size {size}, round {round}" + ); + } } } } diff --git a/crates/common/crypto/src/bls/hashed_messages.rs b/crates/common/crypto/src/bls/hashed_messages.rs new file mode 100644 index 000000000..c17171734 --- /dev/null +++ b/crates/common/crypto/src/bls/hashed_messages.rs @@ -0,0 +1,191 @@ +//! A cache of hashed messages, so a signing root is hashed to G2 once. +//! +//! Hashing a 32-byte signing root to a curve point costs about a hundred +//! microseconds, as much as several of a pairing's Miller loops, and the same +//! root is signed many times: every attester of a slot signs one, and it is +//! checked once off gossip and again inside the block that carries it. +//! +//! The cache is two generations, after the same cache in gattaca-com/silver. +//! New hashes go into `live`. When `live` reaches the cap it becomes `prev` +//! and `live` starts empty, dropping the generation before. A lookup reads +//! both and promotes a `prev` hit back into `live`, so a root that keeps being +//! asked for survives every swap, while one that stops is gone within two +//! generations. That bounds memory at twice the cap without tracking recency +//! per entry, and a hit takes only a read lock unless it is a promotion. +//! +//! The answer is a pure function of the root (and the fixed DST), so sharing +//! one process-wide copy between the state transition, fork choice and gossip +//! validation cannot change a verdict. + +use std::collections::HashMap; +use std::sync::{LazyLock, RwLock}; + +use blst::blst_p2_affine; + +use ethlambda_types::beacon::primitives::Root; + +use super::primitives; + +/// How many hashed messages one generation holds. Silver's value: a few slots' +/// worth of distinct attestation data and block roots, with room for sync +/// committee and aggregate-selection roots. +const MAX_HASHED_MESSAGES: usize = 256; + +/// Every message hashed so far that is still in the two newest generations. +static HASHED_MESSAGES: LazyLock = + LazyLock::new(|| HashedMessages::new(MAX_HASHED_MESSAGES)); + +/// `root` hashed to G2 under the BLS DST, from the process-wide cache. +pub(super) fn hashed_message(root: &Root) -> blst_p2_affine { + HASHED_MESSAGES.get(root) +} + +#[derive(Default)] +struct Generations { + live: HashMap, + prev: HashMap, +} + +/// The two-generation cache; see the module documentation. +struct HashedMessages { + /// The most entries `live` takes before it is retired into `prev`. + cap: usize, + generations: RwLock, +} + +impl HashedMessages { + fn new(cap: usize) -> Self { + Self { + cap, + generations: RwLock::default(), + } + } + + /// The hash of `root`, computed (outside any lock) on a miss. + fn get(&self, root: &Root) -> blst_p2_affine { + let promotable = { + let generations = self.generations.read().unwrap(); + if let Some(point) = generations.live.get(root) { + crate::metrics::inc_hashed_message_lookups(true); + return *point; + } + generations.prev.get(root).copied() + }; + if let Some(point) = promotable { + crate::metrics::inc_hashed_message_lookups(true); + self.insert(*root, point); + return point; + } + crate::metrics::inc_hashed_message_lookups(false); + let point = primitives::hash_to_g2(root.as_slice()); + self.insert(*root, point); + point + } + + /// Adds `root` to `live`, retiring `live` into `prev` first if it is full. + /// A racing writer of the same root leaves the first entry in place. + fn insert(&self, root: Root, point: blst_p2_affine) { + let mut generations = self.generations.write().unwrap(); + if generations.live.contains_key(&root) { + return; + } + if generations.live.len() >= self.cap { + generations.prev = std::mem::take(&mut generations.live); + } + generations.live.insert(root, point); + } + + /// Whether `root` is held in either generation. + #[cfg(test)] + fn holds(&self, root: &Root) -> bool { + let generations = self.generations.read().unwrap(); + generations.live.contains_key(root) || generations.prev.contains_key(root) + } + + /// Whether `root` is in the newest generation. + #[cfg(test)] + fn is_live(&self, root: &Root) -> bool { + self.generations.read().unwrap().live.contains_key(root) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root(n: u64) -> Root { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&n.to_le_bytes()); + Root::from(bytes) + } + + fn bytes_of(point: &blst_p2_affine) -> [u8; 96] { + blst::min_pk::Signature::from(*point).to_bytes() + } + + #[test] + fn a_hit_returns_the_point_a_fresh_hash_does() { + let message = root(1); + let fresh = primitives::hash_to_g2(message.as_slice()); + let first = hashed_message(&message); + let second = hashed_message(&message); + assert_eq!(bytes_of(&first), bytes_of(&fresh)); + assert_eq!(bytes_of(&second), bytes_of(&fresh)); + } + + #[test] + fn the_cached_hash_is_the_one_blst_signs_under() { + let key = blst::min_pk::SecretKey::key_gen(&[7u8; 32], &[]).unwrap(); + let message = root(2); + let signature = key.sign(message.as_slice(), super::super::DST, &[]); + let hashed = hashed_message(&message); + let pk = blst::blst_p1_affine::from(key.sk_to_pk()); + let sig = blst_p2_affine::from(signature); + assert!(primitives::pairing_check(&[(&pk, &hashed)], &sig)); + } + + #[test] + fn a_hot_root_survives_generation_swaps() { + let cache = HashedMessages::new(8); + let hot = root(1_000); + cache.get(&hot); + for n in 0..40 { + cache.get(&root(n)); + cache.get(&hot); + assert!(cache.holds(&hot), "hot root dropped after {n} other roots"); + } + } + + #[test] + fn a_cold_root_is_dropped_after_two_generations() { + let cache = HashedMessages::new(4); + let cold = root(1_000); + cache.get(&cold); + for n in 0..9 { + cache.get(&root(n)); + } + assert!(!cache.holds(&cold)); + } + + #[test] + fn a_hit_in_the_previous_generation_is_promoted() { + let cache = HashedMessages::new(2); + let kept = root(1); + cache.get(&kept); + cache.get(&root(2)); + cache.get(&root(3)); // retires {kept, 2} into prev + assert!(cache.holds(&kept) && !cache.is_live(&kept)); + cache.get(&kept); + assert!(cache.is_live(&kept)); + } + + #[test] + fn the_generations_stay_within_the_cap() { + let cache = HashedMessages::new(8); + for n in 0..100 { + cache.get(&root(n)); + } + let generations = cache.generations.read().unwrap(); + assert!(generations.live.len() <= 8 && generations.prev.len() <= 8); + } +} diff --git a/crates/common/crypto/src/bls/primitives.rs b/crates/common/crypto/src/bls/primitives.rs new file mode 100644 index 000000000..47a9dccd7 --- /dev/null +++ b/crates/common/crypto/src/bls/primitives.rs @@ -0,0 +1,145 @@ +//! The raw `blst` point and pairing calls the BLS layer needs, in one place. +//! +//! Everything above this module speaks in [`PublicKey`], [`Signature`] and +//! [`Root`](ethlambda_types::beacon::primitives::Root); the few operations +//! `blst`'s own verification helpers do not expose as separate steps are here: +//! hashing a message to a G2 point, checking a product of pairings over points +//! that are already hashed, and the weighted sums a random-linear-combination +//! batch needs. Keeping them behind a few small functions leaves one place to +//! re-point when the backend changes, and the only `unsafe` in the BLS layer. +//! +//! The reason these are not `blst`'s `Signature::verify` and friends: those +//! hash their message on every call, so a message signed by many validators +//! (every attester of a slot signs the same root) is hashed once per check. +//! Splitting hashing out lets [`super::hashed_messages`] pay for it once. + +use blst::min_pk::{AggregatePublicKey, AggregateSignature, PublicKey, Signature}; +use blst::{ + BLST_ERROR, MultiPoint, Pairing, blst_fp12, blst_hash_to_g2, blst_p1_affine, blst_p2, + blst_p2_affine, blst_p2_to_affine, +}; +use rayon::prelude::*; + +use super::DST; + +/// How many bits of each batch weight the weighted sums use: the width of the +/// `u64` weights the caller draws. +const WEIGHT_BITS: usize = u64::BITS as usize; + +/// The bit of a compressed point's first byte that marks the point at +/// infinity, the same position in G1 and G2 encodings. +const INFINITY_FLAG: u8 = 0b0100_0000; + +/// Whether a public key is the point at infinity. Only a sum of keys can be: +/// a registry key that is the identity fails `key_validate`. +pub(super) fn key_is_identity(key: &PublicKey) -> bool { + key.to_bytes()[0] & INFINITY_FLAG != 0 +} + +/// Whether a signature is the point at infinity. +pub(super) fn signature_is_identity(signature: &Signature) -> bool { + signature.to_bytes()[0] & INFINITY_FLAG != 0 +} + +/// `message` hashed to a G2 point under [`DST`], in affine form (the form the +/// pairing takes), with the empty augmentation the ciphersuite uses. +pub(super) fn hash_to_g2(message: &[u8]) -> blst_p2_affine { + let mut point = blst_p2::default(); + let mut affine = blst_p2_affine::default(); + let no_aug: &[u8] = &[]; + // SAFETY: `point` and `affine` are valid, exclusively borrowed outputs; + // the message, DST and aug pointers are each valid for the length passed + // with them (an empty slice's pointer is non-null and never read at + // length zero). + unsafe { + blst_hash_to_g2( + &mut point, + message.as_ptr(), + message.len(), + DST.as_ptr(), + DST.len(), + no_aug.as_ptr(), + no_aug.len(), + ); + blst_p2_to_affine(&mut affine, &point); + } + affine +} + +/// Whether `prod_i e(key_i, hashed_i) == e(G1, signature)`, for `pairs` of +/// `(key, hashed message)`. +/// +/// Refuses an empty `pairs` and any identity key: a pairing with the identity +/// contributes nothing, so a hand-built check would let a sum of cancelling +/// keys drop out of the equation instead of failing it (`blst`'s own verify +/// refuses it with `BLST_PK_IS_INFINITY`). Neither keys nor signature are +/// validated here; callers pass points that already passed their checks. +/// +/// With more than one pair the Miller loops are split across rayon's workers, +/// one [`Pairing`] per chunk, merged before the single final exponentiation. +/// The check `blst`'s batch ran was parallel on its own pool, one set per +/// worker, and grouping several pairs on one thread made a block's batch +/// slower than it. +pub(super) fn pairing_check( + pairs: &[(&blst_p1_affine, &blst_p2_affine)], + signature: &blst_p2_affine, +) -> bool { + if pairs.is_empty() + || pairs + .iter() + .any(|(key, _)| key_is_identity(&PublicKey::from(**key))) + { + return false; + } + let accumulate = |chunk: &[(&blst_p1_affine, &blst_p2_affine)]| { + let mut pairing = Pairing::new(true, DST); + for (key, hashed) in chunk { + pairing.raw_aggregate(hashed, key); + } + pairing.commit(); + pairing + }; + // One Miller loop costs about as much as hashing a message, so every pair + // is worth a task: the pairs are spread over at most one chunk per worker. + let mut accumulators: Vec = if pairs.len() == 1 { + vec![accumulate(pairs)] + } else { + let chunk_len = pairs.len().div_ceil(rayon::current_num_threads()); + pairs.par_chunks(chunk_len).map(accumulate).collect() + }; + let Some(mut total) = accumulators.pop() else { + return false; + }; + for other in &accumulators { + if total.merge(other) != BLST_ERROR::BLST_SUCCESS { + return false; + } + } + let mut signature_side = blst_fp12::default(); + Pairing::aggregated(&mut signature_side, signature); + total.finalverify(Some(&signature_side)) +} + +/// The weights as the consecutive little-endian scalars `MultiPoint::mult` +/// reads. +fn scalar_bytes(weights: &[u64]) -> Vec { + weights.iter().flat_map(|w| w.to_le_bytes()).collect() +} + +/// `sum_i weights[i] * keys[i]`, one multi-scalar multiplication. +/// +/// `keys` must be non-empty and the same length as `weights`. +pub(super) fn weighted_key_sum(keys: &[PublicKey], weights: &[u64]) -> PublicKey { + debug_assert!(!keys.is_empty() && keys.len() == weights.len()); + let sum: AggregatePublicKey = keys.mult(&scalar_bytes(weights), WEIGHT_BITS); + sum.to_public_key() +} + +/// `sum_i weights[i] * signatures[i]`, one multi-scalar multiplication. +/// +/// `signatures` must be non-empty and the same length as `weights`. +pub(super) fn weighted_signature_sum(signatures: &[Signature], weights: &[u64]) -> Signature { + debug_assert!(!signatures.is_empty() && signatures.len() == weights.len()); + let sum: AggregateSignature = signatures.mult(&scalar_bytes(weights), WEIGHT_BITS); + sum.to_signature() +} diff --git a/crates/common/crypto/src/bls/warm.rs b/crates/common/crypto/src/bls/warm.rs new file mode 100644 index 000000000..ad6e962aa --- /dev/null +++ b/crates/common/crypto/src/bls/warm.rs @@ -0,0 +1,116 @@ +//! Validating a whole registry's public keys at once, when a state is loaded. +//! +//! `KeyValidate` costs tens of microseconds per key, so a mainnet-sized +//! registry validated lazily taxes the first block imported after a start +//! with every key its committees touch. Doing the whole registry up front, in +//! parallel, moves that cost to load time, where nothing waits on it. + +use blst::min_pk::PublicKey; +use rayon::prelude::*; + +use super::{IndexedPubkey, VALIDATED_PUBKEYS}; + +/// Keys one rayon task validates before it takes the next chunk: large enough +/// that scheduling is noise next to the validations, small enough that the +/// work still spreads across every core. +const CHUNK_LEN: usize = 256; + +/// Validates and caches every key in `pubkeys`, returning how many passed +/// `KeyValidate`, whether cached before the call or by it. +/// +/// A key that fails is skipped and left uncached, so a later lookup of it +/// still fails exactly as it would have without this call; it does not stop +/// the others. Keys already cached are counted without being validated again. +pub fn warm_pubkeys(pubkeys: &[IndexedPubkey]) -> usize { + warm(pubkeys).0 +} + +/// One pass over `pubkeys`: how many keys passed (cached before or by the +/// pass) and how many it ran `KeyValidate` on, so a test can tell a cache hit +/// from a fresh validation. +fn warm(pubkeys: &[IndexedPubkey]) -> (usize, usize) { + pubkeys + .par_chunks(CHUNK_LEN) + .map(|chunk| { + let mut passed = 0; + let mut validated = 0; + for key in chunk { + if VALIDATED_PUBKEYS.get(key).is_some() { + passed += 1; + continue; + } + validated += 1; + if let Ok(point) = PublicKey::key_validate(key.pubkey.as_ref()) { + VALIDATED_PUBKEYS.insert(key, point); + passed += 1; + } + } + (passed, validated) + }) + .reduce(|| (0, 0), |a, b| (a.0 + b.0, a.1 + b.1)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ethlambda_types::beacon::primitives::BlsPubkey; + + fn pubkey(seed: u64) -> BlsPubkey { + let mut ikm = [0x3cu8; 32]; + ikm[..8].copy_from_slice(&seed.to_le_bytes()); + let secret = blst::min_pk::SecretKey::key_gen(&ikm, &[]) + .expect("32 bytes of input material is enough for key generation"); + BlsPubkey(secret.sk_to_pk().to_bytes()) + } + + fn indexed(index: u64, seed: u64) -> IndexedPubkey { + IndexedPubkey { + index, + pubkey: pubkey(seed), + } + } + + #[test] + fn valid_keys_are_cached_and_found_by_index() { + let keys: Vec<_> = (0..600) + .map(|i| indexed(3_100_000 + i, 70_000 + i)) + .collect(); + assert_eq!(warm_pubkeys(&keys), keys.len()); + for key in &keys { + assert!(VALIDATED_PUBKEYS.get(key).is_some()); + // The index table answered, not the by-bytes map. + assert!(VALIDATED_PUBKEYS.get_by_bytes(&key.pubkey).is_none()); + } + } + + #[test] + fn an_invalid_key_is_skipped_without_failing_its_neighbours() { + let invalid = IndexedPubkey { + index: 3_200_001, + pubkey: BlsPubkey::default(), + }; + let keys = [ + indexed(3_200_000, 71_000), + invalid, + indexed(3_200_002, 71_002), + ]; + assert_eq!(warm_pubkeys(&keys), 2); + assert!(VALIDATED_PUBKEYS.get(&keys[0]).is_some()); + assert!(VALIDATED_PUBKEYS.get(&keys[1]).is_none()); + assert!(VALIDATED_PUBKEYS.get(&keys[2]).is_some()); + } + + #[test] + fn an_already_cached_key_is_counted_and_not_validated_again() { + let keys = [indexed(3_300_000, 72_000), indexed(3_300_001, 72_001)]; + let first = warm(&keys); + assert_eq!(first, (2, 2)); + let second = warm(&keys); + assert_eq!(second, (2, 0)); + } + + #[test] + fn empty_input_returns_zero() { + assert_eq!(warm_pubkeys(&[]), 0); + } +} diff --git a/crates/common/crypto/src/metrics.rs b/crates/common/crypto/src/metrics.rs index 997ef3fa2..529b32879 100644 --- a/crates/common/crypto/src/metrics.rs +++ b/crates/common/crypto/src/metrics.rs @@ -72,3 +72,31 @@ pub(crate) fn inc_pubkey_cache_table_entries() { pub(crate) fn inc_pubkey_cache_index_chunks() { LEAN_BEACON_PUBKEY_CACHE_INDEX_CHUNKS.inc(); } + +static LEAN_BEACON_HASHED_MESSAGE_CACHE_LOOKUPS_TOTAL: LazyLock = + LazyLock::new(|| { + register_int_counter_vec!( + "lean_beacon_hashed_message_cache_lookups_total", + "Signing roots resolved to a G2 point for a signature check, by whether the \ + hashed-message cache already held the hash", + &["result"] + ) + .unwrap() + }); + +// Resolved once for the same reason as the pubkey cache's: a lookup is counted +// on every signature check. +static LEAN_BEACON_HASHED_MESSAGE_CACHE_HITS: LazyLock = + LazyLock::new(|| LEAN_BEACON_HASHED_MESSAGE_CACHE_LOOKUPS_TOTAL.with_label_values(&["hit"])); +static LEAN_BEACON_HASHED_MESSAGE_CACHE_MISSES: LazyLock = + LazyLock::new(|| LEAN_BEACON_HASHED_MESSAGE_CACHE_LOOKUPS_TOTAL.with_label_values(&["miss"])); + +/// Count one signing root looked up in the hashed-message cache: a hit found +/// it in either generation, a miss hashed it. +pub(crate) fn inc_hashed_message_lookups(hit: bool) { + if hit { + LEAN_BEACON_HASHED_MESSAGE_CACHE_HITS.inc(); + } else { + LEAN_BEACON_HASHED_MESSAGE_CACHE_MISSES.inc(); + } +} diff --git a/crates/net/p2p/Cargo.toml b/crates/net/p2p/Cargo.toml index 5923e6e40..8030bf96c 100644 --- a/crates/net/p2p/Cargo.toml +++ b/crates/net/p2p/Cargo.toml @@ -23,6 +23,9 @@ ethlambda-state-transition.workspace = true spawned-concurrency.workspace = true +# Spreads the work inside one gossip batch across cores; see `beacon::batched`. +rayon.workspace = true + futures = "0.3" tokio-stream = "0.1" @@ -87,3 +90,8 @@ libssz-types.workspace = true sha2 = "0.10" hex.workspace = true + +[dev-dependencies] +# The Beacon state builder and its signing keys, so the batched gossip checks +# can be run against states that verify real signatures. +ethlambda-state-transition = { workspace = true, features = ["test-utils"] } diff --git a/crates/net/p2p/src/beacon/batched.rs b/crates/net/p2p/src/beacon/batched.rs new file mode 100644 index 000000000..d28d6fe88 --- /dev/null +++ b/crates/net/p2p/src/beacon/batched.rs @@ -0,0 +1,812 @@ +//! Batched stateful checks for the gossip kinds that arrive in bursts and cost +//! a pairing or a KZG proof each: subnet attestations and aggregates, sync +//! committee messages, and data column sidecars. +//! +//! Verifying them one by one pays each pairing's final exponentiation (and +//! each KZG call's fixed work) once per message. Here a message is queued +//! instead, and whichever blocking task is draining takes everything queued +//! and verifies it as one batch: every deferred signature of every message in +//! a single [`bls::verify_signature_sets`] call (one +//! `verify_cell_kzg_proof_batch` call across all the sidecars for columns). +//! Only when the batch fails is each message checked on its own, which yields +//! exactly the verdicts the one-by-one path yields: the batch is an +//! optimization over a pass that is almost always all-valid, never a different +//! rule. +//! +//! # Queue protocol +//! +//! There is no standing worker. A [`Batcher`] holds a queue and a `draining` +//! flag under one lock. The enqueue that finds no drainer running takes a +//! permit from the kind's usual pool and spawns one; every other enqueue just +//! pushes. The drainer loops, taking up to [`BATCH_CAP`] items at a time, and +//! clears the flag under the same lock that observes the queue empty, so a +//! message pushed after that starts a new drainer instead of waiting on one +//! that is already leaving. +//! +//! A drainer holds its permit for its whole run, and the `draining` flag +//! allows one drainer per kind at a time: that single drainer is the batching +//! point, since everything that arrives while it works lands in the next +//! batch. The work inside a batch is not serial, though: preparing each +//! message and, after a failed batch, checking each message on its own are +//! spread across cores with rayon, and the one combined crypto call +//! parallelises internally. A message that cannot be queued +//! (the queue is at [`QUEUE_CAP`], or it needs a drainer and the pool has no +//! permit) is answered `Ignore(Overloaded)` at once, as a message with no +//! permit always was: it is never held past the time gossipsub can wait. + +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; + +use ethlambda_state_transition::beacon::bls::{self, SignatureSet}; +use ethlambda_state_transition::beacon::gossip::column::{self, ColumnPrepared}; +use ethlambda_state_transition::beacon::gossip::{self, IgnoreReason, Outcome}; +use ethlambda_storage::Store; +use ethlambda_types::beacon::containers::DataColumnSidecar; +use ethlambda_types::beacon::primitives::ValidatorIndex; +use rayon::prelude::*; +use spawned_concurrency::tasks::{ActorRef, Context}; +use tracing::warn; + +use crate::beacon::verdict::{self, GossipId, GossipVerdict, Validated, guarded_try}; +use crate::{P2PServer, metrics}; + +/// The most messages one drain pass verifies together. Large enough that the +/// fixed cost of a batch is spread thin; small enough that a forged message +/// costs the rest of its batch one fallback pass rather than an unbounded one. +const BATCH_CAP: usize = 1024; + +/// The most messages that may wait for a drainer. Past it a message is +/// answered `Ignore(Overloaded)`: a longer queue would only make verdicts +/// later than gossipsub's cache can wait for. +const QUEUE_CAP: usize = 4 * BATCH_CAP; + +/// A message waiting to be verified. +pub(crate) struct Queued { + id: GossipId, + object: Validated, +} + +/// What [`Batcher::push`] decided. +pub(crate) enum Push { + /// Queued behind a drainer that is already running. + Queued, + /// Queued, and no drainer was running: the caller spawns one, holding the + /// permit it was handed. + StartDrainer(P), + /// Not queued: the queue is full, or a drainer was needed and no permit + /// was free. The item comes back so the caller can answer it. + Overloaded(T), +} + +struct QueueState { + items: VecDeque, + /// Whether a drainer is running, or about to be (its permit is taken and + /// its task is being spawned). + draining: bool, +} + +/// A queue plus the "a drainer is running" flag, behind one lock. +pub(crate) struct Batcher { + state: Mutex>, + queue_cap: usize, + batch_cap: usize, +} + +impl Batcher { + pub(crate) fn new(queue_cap: usize, batch_cap: usize) -> Self { + Self { + state: Mutex::new(QueueState { + items: VecDeque::new(), + draining: false, + }), + queue_cap, + batch_cap, + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, QueueState> { + // Nothing under the lock can panic midway, so a poisoned lock still + // holds a consistent queue. + self.state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// Queue `item`. `acquire` takes the permit a new drainer needs, and is + /// only called when none is running. It runs under the lock, so it must + /// not block: a semaphore's `try_acquire` is what it is for. + pub(crate) fn push

(&self, item: T, acquire: impl FnOnce() -> Option

) -> Push { + let mut state = self.lock(); + if state.items.len() >= self.queue_cap { + return Push::Overloaded(item); + } + if state.draining { + state.items.push_back(item); + return Push::Queued; + } + let Some(permit) = acquire() else { + return Push::Overloaded(item); + }; + state.draining = true; + state.items.push_back(item); + Push::StartDrainer(permit) + } + + /// The next batch, or `None` once the queue is empty, which also marks + /// the drainer as gone, under the lock the emptiness was seen under. + pub(crate) fn take_batch(&self) -> Option> { + let mut state = self.lock(); + if state.items.is_empty() { + state.draining = false; + return None; + } + let take = state.items.len().min(self.batch_cap); + Some(state.items.drain(..take).collect()) + } + + /// Mark the drainer gone without the queue having been seen empty, for a + /// drainer that is unwinding: the next push then starts a new one, which + /// takes whatever is still queued. + fn abandon(&self) { + self.lock().draining = false; + } +} + +/// Clears the `draining` flag if the drainer unwinds, so a panic cannot leave +/// the queue without a drainer for ever. +struct DrainGuard<'a, T>(&'a Batcher); + +impl Drop for DrainGuard<'_, T> { + fn drop(&mut self) { + if std::thread::panicking() { + self.0.abandon(); + } + } +} + +/// The three batched kinds' queues, one per permit pool they draw from. +pub(crate) struct Batchers { + /// Subnet attestations and aggregates, from the attestation pool. + votes: Batcher, + /// Sync committee messages, from the sync pool. + sync: Batcher, + /// Data column sidecars, from the gossip pool. + columns: Batcher, +} + +impl Default for Batchers { + fn default() -> Self { + Self { + votes: Batcher::new(QUEUE_CAP, BATCH_CAP), + sync: Batcher::new(QUEUE_CAP, BATCH_CAP), + columns: Batcher::new(QUEUE_CAP, BATCH_CAP), + } + } +} + +/// Which batcher an object goes through. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Kind { + Votes, + Sync, + Columns, +} + +impl Kind { + /// `None` for every object that keeps the per-message path. + pub(crate) fn of(object: &Validated) -> Option { + match object { + Validated::Attestation { .. } | Validated::Aggregate { .. } => Some(Self::Votes), + Validated::SyncCommitteeMessage { .. } => Some(Self::Sync), + Validated::Column(_) => Some(Self::Columns), + Validated::Block { .. } + | Validated::Envelope(_) + | Validated::ExecutionPayloadBid { .. } + | Validated::ProposerPreferences(_) + | Validated::PayloadAttestation(_) + | Validated::Operation(_) + | Validated::SyncContribution(_) => None, + } + } + + /// The metric label. + fn label(self) -> &'static str { + match self { + Self::Votes => "votes", + Self::Sync => "sync_committee", + Self::Columns => "column", + } + } + + fn batcher(self, batchers: &Batchers) -> &Batcher { + match self { + Self::Votes => &batchers.votes, + Self::Sync => &batchers.sync, + Self::Columns => &batchers.columns, + } + } +} + +/// Queue `object` for batched verification, which the caller has found to be +/// of batched `kind`. +pub(crate) fn enqueue( + server: &P2PServer, + ctx: &Context, + kind: Kind, + id: GossipId, + object: Validated, +) { + let pool = verdict::pool_of(&object); + let permits = verdict::permits_for(server, &object).clone(); + let batchers = Arc::clone(&server.batchers); + let pushed = kind + .batcher(&batchers) + .push(Queued { id, object }, || permits.try_acquire_owned().ok()); + match pushed { + Push::Queued => {} + Push::Overloaded(Queued { id, object }) => { + verdict::report_overloaded(server, pool, id, object) + } + Push::StartDrainer(permit) => { + let store = server.store.clone(); + let actor = ctx.actor_ref(); + let in_use = metrics::PoolInUse::new(pool); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let _in_use = in_use; + drain(kind.batcher(&batchers), kind, &store, &actor); + }); + } + } +} + +/// The drainer: verify batches until the queue is empty. It is the only one +/// running for its kind, which is what makes a batch; the per-message work +/// inside [`verify_batch`] is spread across cores with rayon. +fn drain(batcher: &Batcher, kind: Kind, store: &Store, actor: &ActorRef) { + let _guard = DrainGuard(batcher); + while let Some(batch) = batcher.take_batch() { + let (ids, mut objects): (Vec, Vec) = batch + .into_iter() + .map(|queued| (queued.id, queued.object)) + .unzip(); + let outcomes = verify_batch(store, kind, &mut objects); + for ((id, object), outcome) in ids.into_iter().zip(objects).zip(outcomes) { + let _ = actor + .send(GossipVerdict::new(id, outcome, object)) + .inspect_err(|_| warn!("P2P actor stopped before a gossip verdict arrived")); + } + } +} + +/// One signature-checked object's stateful checks, evaluated up to its +/// pairings. +enum Prepared { + Attestation(gossip::Prepared<()>), + Aggregate(gossip::Prepared>), + Sync(gossip::Prepared>), + /// Already decided: the preparation panicked, or the object is of a kind + /// this batch does not verify. + Done(Outcome), +} + +impl Prepared { + fn new(store: &Store, object: &Validated) -> Self { + match object { + Validated::Attestation { + attestation, + subnet_id, + } => Self::Attestation(gossip::attestation::prepare_stateful_checks( + store, + attestation, + *subnet_id, + )), + Validated::Aggregate { aggregate, .. } => { + Self::Aggregate(gossip::aggregate::prepare_stateful_checks(store, aggregate)) + } + Validated::SyncCommitteeMessage { + message, subnet_id, .. + } => Self::Sync(gossip::sync_committee::prepare_message_stateful_checks( + store, message, *subnet_id, + )), + _ => Self::Done(Outcome::Ignore(IgnoreReason::Internal)), + } + } + + /// The pairings this object needs verified for its verdict to be settled. + fn signature_sets(&self) -> Vec { + match self { + Self::Attestation(prepared) => prepared.signature_sets(), + Self::Aggregate(prepared) => prepared.signature_sets(), + Self::Sync(prepared) => prepared.signature_sets(), + Self::Done(_) => Vec::new(), + } + } + + /// The verdict, writing what `Accept` carries into `object`. + /// `batch_passed` says every pairing of the batch verified; when it did + /// not, each pairing is evaluated here, on its own. + fn resolve(self, batch_passed: bool, object: &mut Validated) -> Outcome { + match (self, object) { + (Self::Attestation(prepared), _) => accept_or(resolve(prepared, batch_passed)), + ( + Self::Aggregate(prepared), + Validated::Aggregate { + attesting_indices, .. + }, + ) => match resolve(prepared, batch_passed) { + Ok(indices) => { + *attesting_indices = indices; + Outcome::Accept + } + Err(outcome) => outcome, + }, + (Self::Sync(prepared), Validated::SyncCommitteeMessage { seats, .. }) => { + match resolve(prepared, batch_passed) { + Ok(resolved) => { + *seats = resolved; + Outcome::Accept + } + Err(outcome) => outcome, + } + } + (Self::Done(outcome), _) => outcome, + // `Prepared::new` built each variant from the object it is now + // resolved against. + (Self::Aggregate(_) | Self::Sync(_), _) => Outcome::Ignore(IgnoreReason::Internal), + } + } +} + +fn accept_or(resolved: Result<(), Outcome>) -> Outcome { + resolved.map_or_else(|outcome| outcome, |()| Outcome::Accept) +} + +fn resolve(prepared: gossip::Prepared, batch_passed: bool) -> Result { + if batch_passed { + prepared.resolve_batch_passed() + } else { + prepared.resolve() + } +} + +/// Verify `objects` as one batch and return one verdict per object, in order, +/// filling in what an `Accept` carries (an aggregate's attesting indices, a +/// sync message's seats). +/// +/// Preparation runs per object inside a panic guard, so a panic is +/// `Ignore(Internal)` for that object alone. The batched crypto call is +/// guarded too: a panic in it counts as a failed batch and falls back to +/// per-object checks. +pub(crate) fn verify_batch(store: &Store, kind: Kind, objects: &mut [Validated]) -> Vec { + metrics::observe_gossip_batch_size(kind.label(), objects.len()); + if kind == Kind::Columns { + return verify_columns(store, objects); + } + let prepared: Vec = objects + .par_iter() + .map(|object| guarded_try(|| Prepared::new(store, object)).unwrap_or_else(Prepared::Done)) + .collect(); + let sets: Vec = prepared.iter().flat_map(Prepared::signature_sets).collect(); + let batch_passed = guarded_try(|| bls::verify_signature_sets(&sets)).unwrap_or(false); + if !batch_passed { + metrics::inc_gossip_batch_fallback(kind.label()); + } + prepared + .into_par_iter() + .zip(objects.par_iter_mut()) + .map(|(item, object)| { + guarded_try(|| item.resolve(batch_passed, object)).unwrap_or_else(|outcome| outcome) + }) + .collect() +} + +/// The column half of [`verify_batch`]: one `verify_cell_kzg_proof_batch` call +/// over every pending sidecar, per-sidecar verification only if it fails. +fn verify_columns(store: &Store, objects: &[Validated]) -> Vec { + let sidecars: Vec> = objects + .iter() + .map(|object| match object { + Validated::Column(sidecar) => Some(&**sidecar), + _ => None, + }) + .collect(); + let prepared: Vec = sidecars + .par_iter() + .map(|sidecar| match sidecar { + Some(sidecar) => guarded_try(|| column::prepare_stateful_checks(store, sidecar)) + .unwrap_or_else(ColumnPrepared::Settled), + None => ColumnPrepared::Settled(Outcome::Ignore(IgnoreReason::Internal)), + }) + .collect(); + let pending: Vec<(&DataColumnSidecar, &column::PendingKzg)> = prepared + .iter() + .zip(&sidecars) + .filter_map(|(item, sidecar)| Some(((*sidecar)?, item.pending()?))) + .collect(); + let batch_passed = + pending.is_empty() || guarded_try(|| column::kzg_batch_verifies(&pending)).unwrap_or(false); + // A lone sidecar's batch is its own check: a failure needs no second pass. + let fall_back = !batch_passed && pending.len() > 1; + if fall_back { + metrics::inc_gossip_batch_fallback(Kind::Columns.label()); + } + prepared + .into_par_iter() + .zip(sidecars.par_iter()) + .map(|(item, sidecar)| { + let proofs_verify = match (item.pending(), sidecar) { + (None, _) => true, + (Some(_), _) if batch_passed => true, + (Some(_), _) if !fall_back => false, + (Some(pending), Some(sidecar)) => { + guarded_try(|| column::kzg_proofs_verify(sidecar, pending)).unwrap_or(false) + } + (Some(_), None) => false, + }; + item.settle(proofs_verify) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + fn batcher(queue_cap: usize, batch_cap: usize) -> Batcher { + Batcher::new(queue_cap, batch_cap) + } + + #[test] + fn the_first_push_starts_a_drainer_and_later_ones_queue_behind_it() { + let batcher = batcher(8, 8); + assert!(matches!( + batcher.push(1, || Some(())), + Push::StartDrainer(()) + )); + assert!(matches!( + batcher.push(2, || -> Option<()> { panic!("no second permit") }), + Push::Queued + )); + assert_eq!(batcher.take_batch(), Some(vec![1, 2])); + } + + #[test] + fn a_push_without_a_permit_is_overloaded_and_queues_nothing() { + let batcher = batcher(8, 8); + assert!(matches!( + batcher.push(1, || None::<()>), + Push::Overloaded(1) + )); + // Nothing was queued and no drainer is recorded: the next push may + // still start one. + assert!(matches!( + batcher.push(2, || Some(())), + Push::StartDrainer(()) + )); + assert_eq!(batcher.take_batch(), Some(vec![2])); + } + + #[test] + fn a_full_queue_answers_overloaded() { + let batcher = batcher(2, 8); + assert!(matches!( + batcher.push(1, || Some(())), + Push::StartDrainer(()) + )); + assert!(matches!(batcher.push(2, || Some(())), Push::Queued)); + assert!(matches!(batcher.push(3, || Some(())), Push::Overloaded(3))); + // Taking a batch frees room again. + assert_eq!(batcher.take_batch(), Some(vec![1, 2])); + assert!(matches!(batcher.push(4, || Some(())), Push::Queued)); + } + + #[test] + fn a_batch_is_at_most_the_batch_cap_and_keeps_arrival_order() { + let batcher = batcher(16, 3); + for item in 0..7 { + batcher.push(item, || Some(())); + } + assert_eq!(batcher.take_batch(), Some(vec![0, 1, 2])); + assert_eq!(batcher.take_batch(), Some(vec![3, 4, 5])); + assert_eq!(batcher.take_batch(), Some(vec![6])); + assert_eq!(batcher.take_batch(), None); + } + + #[test] + fn a_message_pushed_after_the_drainer_saw_an_empty_queue_starts_a_new_drainer() { + let batcher = batcher(8, 8); + batcher.push(1, || Some(())); + assert_eq!(batcher.take_batch(), Some(vec![1])); + assert_eq!(batcher.take_batch(), None); + // The drainer has left; this one must not wait behind it. + assert!(matches!( + batcher.push(2, || Some(())), + Push::StartDrainer(()) + )); + assert_eq!(batcher.take_batch(), Some(vec![2])); + } + + #[test] + fn an_abandoned_drainer_leaves_its_queue_to_the_next_push() { + let batcher = batcher(8, 8); + batcher.push(1, || Some(())); + batcher.abandon(); + assert!(matches!( + batcher.push(2, || Some(())), + Push::StartDrainer(()) + )); + assert_eq!(batcher.take_batch(), Some(vec![1, 2])); + } + + /// Hammer one batcher from several producers while drainers come and go: + /// every item pushed is either rejected on the spot or taken by exactly + /// one batch, and none is left behind once the producers stop. + #[test] + fn every_pushed_item_is_answered_while_drainers_finish_and_restart() { + let batcher = Arc::new(batcher(64, 5)); + let answered = Arc::new(AtomicUsize::new(0)); + let producers = 4; + let per_producer = 500; + let mut handles = Vec::new(); + for producer in 0..producers { + let batcher = Arc::clone(&batcher); + let answered = Arc::clone(&answered); + handles.push(std::thread::spawn(move || { + let mut drainers = Vec::new(); + for item in 0..per_producer { + match batcher.push((producer * per_producer + item) as u32, || Some(())) { + Push::Queued => {} + Push::Overloaded(_) => { + answered.fetch_add(1, Ordering::SeqCst); + } + Push::StartDrainer(()) => { + let batcher = Arc::clone(&batcher); + let answered = Arc::clone(&answered); + drainers.push(std::thread::spawn(move || { + while let Some(batch) = batcher.take_batch() { + answered.fetch_add(batch.len(), Ordering::SeqCst); + } + })); + } + } + } + for drainer in drainers { + drainer.join().expect("drainer"); + } + })); + } + for handle in handles { + handle.join().expect("producer"); + } + // Whatever a drainer left queued when it exited is picked up by a + // push that came after, so nothing is stranded. + assert_eq!(batcher.take_batch(), None); + assert_eq!(answered.load(Ordering::SeqCst), producers * per_producer); + } +} + +/// [`verify_batch`] against states that verify real signatures. +#[cfg(test)] +mod verify_tests { + use ethlambda_state_transition::beacon::constants::DOMAIN_BEACON_ATTESTER; + use ethlambda_state_transition::beacon::fork::ForkName; + use ethlambda_state_transition::beacon::gossip::attestation::compute_subnet_for_attestation; + use ethlambda_state_transition::beacon::gossip::{IgnoreReason, QueueReason, RejectReason}; + use ethlambda_state_transition::beacon::helpers::accessors::{ + CommitteeCacheExt as _, get_beacon_committee, + }; + use ethlambda_state_transition::beacon::helpers::misc::{ + compute_domain, compute_signing_root, compute_start_slot_at_epoch, + }; + use ethlambda_state_transition::beacon::helpers::test_state::{ + sign_for, with_signing_validators_at, + }; + use ethlambda_storage::backend::InMemoryBackend; + use ethlambda_storage::{CacheKey, Store}; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::containers::electra::{self, SingleAttestation}; + use ethlambda_types::beacon::containers::shared::{AttestationData, Checkpoint, Fork}; + use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock}; + use ethlambda_types::beacon::primitives::{H256, HashTreeRoot as _, Root}; + + use super::*; + use crate::test_support::valid_shaped_sidecar; + + const FULU_EPOCH: u64 = 2; + + /// A store holding the vote block's state, `attesters` correctly signed + /// votes from distinct members of one committee, and their subnet. The + /// vote is cast in a fork's first slot with no block, so it is signed + /// under the schedule's domain, as the rules' own tests do. + fn votes(attesters: usize) -> (Store, Vec, u64) { + let config = Config::mainnet() + .with_fork_epoch(ForkName::Electra, 0) + .with_fork_epoch(ForkName::Fulu, FULU_EPOCH); + let mut state = with_signing_validators_at(ForkName::Electra, 256); + let pre_fork_slot = compute_start_slot_at_epoch(FULU_EPOCH) - 1; + *state.slot_mut() = pre_fork_slot; + *state.fork_mut() = Fork { + previous_version: config.deneb_fork_version, + current_version: config.electra_fork_version, + epoch: 0, + }; + state.apply_pending_mutations(); + + let anchor = ethlambda_types::checkpoint::Checkpoint { + root: H256::ZERO, + slot: 0, + }; + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::new()), + config.genesis_time, + config.clone(), + H256::ZERO, + anchor, + 0, + ); + let block_root = Root::repeat_byte(7); + let block = SignedBeaconBlock::Electra(electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot: pre_fork_slot, + proposer_index: 0, + parent_root: Root::ZERO, + state_root: Root::ZERO, + body: electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }); + store + .insert_pending_block(block_root, block) + .expect("insert the voted block"); + store.cache_state(CacheKey::BlockState(block_root), Arc::new(state.clone())); + + let slot = compute_start_slot_at_epoch(FULU_EPOCH); + let committee = get_beacon_committee(&state, slot, 0).expect("committee"); + assert!(committee.len() >= attesters, "the committee is big enough"); + let data = AttestationData { + slot, + index: 0, + beacon_block_root: block_root, + source: Default::default(), + target: Checkpoint { + epoch: FULU_EPOCH, + root: block_root, + }, + }; + let domain = compute_domain( + DOMAIN_BEACON_ATTESTER, + config.fulu_fork_version, + state.genesis_validators_root(), + ); + let signing_root = compute_signing_root(data.hash_tree_root(), domain); + let votes = committee + .iter() + .take(attesters) + .map(|&attester_index| SingleAttestation { + committee_index: 0, + attester_index, + data, + signature: sign_for(attester_index as usize, signing_root), + }) + .collect(); + let committees_per_slot = store + .committee_cache() + .committees(&state, FULU_EPOCH) + .committees_per_slot(); + let subnet_id = compute_subnet_for_attestation(committees_per_slot, slot, 0, &config); + (store, votes, subnet_id) + } + + fn object(attestation: SingleAttestation, subnet_id: u64) -> Validated { + Validated::Attestation { + attestation: Box::new(attestation), + subnet_id, + } + } + + #[test] + fn a_drain_of_valid_attestations_accepts_every_one() { + let (store, votes, subnet_id) = votes(6); + let mut objects: Vec = votes + .into_iter() + .map(|vote| object(vote, subnet_id)) + .collect(); + let outcomes = verify_batch(&store, Kind::Votes, &mut objects); + assert_eq!(outcomes, vec![Outcome::Accept; 6]); + } + + /// The batch fails on the forged signature, and the fallback then settles + /// each message on its own: only the forged one is rejected. + #[test] + fn one_forged_signature_rejects_only_that_attestation() { + let (store, mut votes, subnet_id) = votes(6); + let forged = 3; + votes[forged].signature = sign_for( + votes[forged].attester_index as usize + 1, + votes[forged].data.hash_tree_root(), + ); + let mut objects: Vec = votes + .into_iter() + .map(|vote| object(vote, subnet_id)) + .collect(); + let outcomes = verify_batch(&store, Kind::Votes, &mut objects); + for (position, outcome) in outcomes.into_iter().enumerate() { + if position == forged { + assert_eq!(outcome, Outcome::Reject(RejectReason::BadSignature)); + } else { + assert_eq!(outcome, Outcome::Accept, "message {position}"); + } + } + } + + /// A message that fails a rule before its signature is judged by that rule + /// and takes nothing from the batch; one that fails a rule after a good + /// signature gets that rule's verdict; the rest are accepted. + #[test] + fn rule_failures_in_a_batch_keep_their_own_verdicts() { + let (store, votes, subnet_id) = votes(4); + let mut objects = vec![ + object(votes[0].clone(), subnet_id), + // On the wrong subnet. + object(votes[1].clone(), subnet_id + 1), + // An unknown block: nothing to verify. + object( + { + let mut vote = votes[2].clone(); + vote.data.beacon_block_root = Root::repeat_byte(1); + vote + }, + subnet_id, + ), + object(votes[3].clone(), subnet_id), + ]; + let outcomes = verify_batch(&store, Kind::Votes, &mut objects); + assert_eq!( + outcomes, + vec![ + Outcome::Accept, + Outcome::Reject(RejectReason::WrongSubnet), + Outcome::Ignore(IgnoreReason::UnknownBlock), + Outcome::Accept, + ] + ); + } + + #[test] + fn a_single_message_batch_matches_the_one_by_one_verdict() { + let (store, votes, subnet_id) = votes(1); + let mut objects = vec![object(votes[0].clone(), subnet_id)]; + assert_eq!( + verify_batch(&store, Kind::Votes, &mut objects), + vec![Outcome::Accept] + ); + } + + #[test] + fn an_empty_batch_has_no_verdicts() { + let (store, _, _) = votes(0); + assert!(verify_batch(&store, Kind::Votes, &mut []).is_empty()); + } + + /// Columns whose block is unknown are queued by their stateful rules; the + /// batch has no proofs to verify for them, and each still gets its verdict. + #[test] + fn columns_decided_before_their_proofs_get_their_verdicts() { + let (store, _, _) = votes(0); + let mut objects: Vec = (0..3) + .map(|index| { + Validated::Column(Box::new(DataColumnSidecar::Fulu(valid_shaped_sidecar( + 5, index, + )))) + }) + .collect(); + let outcomes = verify_batch(&store, Kind::Columns, &mut objects); + assert_eq!( + outcomes, + vec![Outcome::Queue(QueueReason::ParentUnknown); 3] + ); + } +} diff --git a/crates/net/p2p/src/beacon/mod.rs b/crates/net/p2p/src/beacon/mod.rs index 17710ee8f..86f41389b 100644 --- a/crates/net/p2p/src/beacon/mod.rs +++ b/crates/net/p2p/src/beacon/mod.rs @@ -13,6 +13,7 @@ //! written once and handed the two things the chains disagree about: how wide //! the `` field is, and how a chunk body becomes a block. +pub mod batched; pub mod builder_market; pub mod column_checks; pub mod decode; diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index 57b67d53e..8bb7fce72 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -9,7 +9,11 @@ //! that field's own documentation for why they must not share one), a sync //! committee message or contribution from //! [`P2PServer::sync_validation_permits`]. Either way -//! the blocking task sends a [`GossipVerdict`] back to the actor. Every beacon +//! the blocking task sends a [`GossipVerdict`] back to the actor. The kinds +//! that arrive in bursts and cost a pairing or a KZG proof each (subnet +//! attestations, aggregates, sync committee messages, data columns) are queued +//! instead and verified in batches by [`crate::beacon::batched`], which draws +//! its drainer's permit from the same pool. Every beacon //! gossip message ends in exactly one [`report`]: gossipsub holds each one //! until then. @@ -42,7 +46,7 @@ use spawned_concurrency::message::Message; use spawned_concurrency::tasks::{Context, Handler}; use tracing::{debug, error, warn}; -use crate::beacon::{builder_market, column_checks}; +use crate::beacon::{batched, builder_market, column_checks}; use crate::gossipsub::operation_kind; use crate::{P2PServer, metrics}; @@ -585,6 +589,16 @@ pub(crate) struct GossipVerdict { object: Validated, } +impl GossipVerdict { + pub(crate) fn new(id: GossipId, outcome: Outcome, object: Validated) -> Self { + Self { + id, + outcome, + object, + } + } +} + impl Message for GossipVerdict { type Result = (); } @@ -653,7 +667,7 @@ pub(crate) fn report(server: &P2PServer, id: GossipId, outcome: Outcome) -> bool /// operation topics) from the attestation pool, and the sync committee's /// messages and contributions from a pool of their own, so no burst starves /// another. -fn permits_for<'a>( +pub(crate) fn permits_for<'a>( server: &'a P2PServer, object: &Validated, ) -> &'a std::sync::Arc { @@ -668,7 +682,7 @@ fn permits_for<'a>( /// The label (see [`metrics::VALIDATION_POOLS`]) of the permit pool /// [`permits_for`] draws `object`'s permit from. The one `match` on the /// object kind, so the pool metrics and the pool actually used cannot differ. -fn pool_of(object: &Validated) -> &'static str { +pub(crate) fn pool_of(object: &Validated) -> &'static str { match object { Validated::Block { .. } | Validated::Column(_) | Validated::Envelope(_) => "gossip", Validated::Aggregate { .. } @@ -705,13 +719,15 @@ pub(crate) fn spawn_stateful_checks( id: GossipId, object: Validated, ) { + // The kinds that arrive in bursts and cost a pairing or a KZG proof each + // are verified in batches; see [`batched`]. + let object = match batched::Kind::of(&object) { + Some(kind) => return batched::enqueue(server, ctx, kind, id, object), + None => object, + }; let pool = pool_of(&object); let Ok(permit) = permits_for(server, &object).clone().try_acquire_owned() else { - metrics::inc_validation_permit_exhausted(pool); - let received_at = id.received_at; - let outcome = Outcome::Ignore(IgnoreReason::Overloaded); - report(server, id, outcome); - object.forward(server, received_at, outcome); + report_overloaded(server, pool, id, object); return; }; let store = server.store.clone(); @@ -734,6 +750,18 @@ pub(crate) fn spawn_stateful_checks( }); } +/// Answer `object` `Ignore(Overloaded)`: its pool had no permit (or its batch +/// queue was full), and holding it for later would only make its verdict +/// arrive after gossipsub's cache has evicted the message. Forwarded as +/// [`Validated::forward`] documents for that outcome. +pub(crate) fn report_overloaded(server: &P2PServer, pool: &str, id: GossipId, object: Validated) { + metrics::inc_validation_permit_exhausted(pool); + let received_at = id.received_at; + let outcome = Outcome::Ignore(IgnoreReason::Overloaded); + report(server, id, outcome); + object.forward(server, received_at, outcome); +} + /// `checks()`, with a panic turned into `Ignore(Internal)`. /// /// A panic inside validation (a `Store` read's `.expect()` on a DB error, say) @@ -741,7 +769,14 @@ pub(crate) fn spawn_stateful_checks( /// cache evicts it. The blocking thread's permit is unaffected either way: it /// drops on unwind exactly as it would on a normal return. pub(crate) fn guarded(checks: impl FnOnce() -> Outcome) -> Outcome { - catch_unwind(AssertUnwindSafe(checks)).unwrap_or_else(|payload| { + guarded_try(checks).unwrap_or_else(|outcome| outcome) +} + +/// `checks()`, with a panic turned into `Err(Ignore(Internal))`: [`guarded`] +/// for a closure that returns more than a verdict, such as one stage of a +/// batch, whose panic must cost only the object it was working on. +pub(crate) fn guarded_try(checks: impl FnOnce() -> T) -> Result { + catch_unwind(AssertUnwindSafe(checks)).map_err(|payload| { let panic_message = payload .downcast_ref::<&str>() .copied() diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 4f670185b..860ab2fc1 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -1260,6 +1260,7 @@ impl P2P { GOSSIP_VALIDATION_PERMITS, )), sync_validation_permits: Arc::new(tokio::sync::Semaphore::new(SYNC_VALIDATION_PERMITS)), + batchers: Default::default(), column_check_permits: Arc::new(tokio::sync::Semaphore::new(COLUMN_CHECK_PERMITS)), attestation_validation_permits: Arc::new(tokio::sync::Semaphore::new( ATTESTATION_VALIDATION_PERMITS, @@ -1455,6 +1456,10 @@ pub struct P2PServer { /// checks in flight on blocking threads. Separate from the other two /// pools; see [`SYNC_VALIDATION_PERMITS`]. pub(crate) sync_validation_permits: Arc, + /// The queues the batched gossip kinds (attestations, aggregates, sync + /// committee messages, data columns) wait in for a drainer; see + /// [`beacon::batched`]. + pub(crate) batchers: Arc, /// Accepted `payload_attestation_message` votes, shared with the Beacon /// API that serves and fills the same pool. Filled by `verdict::forward`; @@ -3164,6 +3169,7 @@ pub(crate) mod test_support { crate::SEEN_SYNC_AGGREGATORS_CAPACITY, crate::SEEN_SYNC_DATA_CAPACITY, ), + batchers: Default::default(), sync_validation_permits: std::sync::Arc::new(tokio::sync::Semaphore::new( crate::SYNC_VALIDATION_PERMITS, )), diff --git a/crates/net/p2p/src/metrics.rs b/crates/net/p2p/src/metrics.rs index df5ab4323..3c73ab4a4 100644 --- a/crates/net/p2p/src/metrics.rs +++ b/crates/net/p2p/src/metrics.rs @@ -454,6 +454,10 @@ pub fn init() { LEAN_BEACON_VALIDATION_PERMITS_IN_USE.with_label_values(&[pool]); LEAN_BEACON_VALIDATION_PERMIT_EXHAUSTED_TOTAL.with_label_values(&[pool]); } + for &kind in BATCHED_KINDS { + LEAN_BEACON_GOSSIP_BATCH_SIZE.with_label_values(&[kind]); + LEAN_BEACON_GOSSIP_BATCH_FALLBACK_TOTAL.with_label_values(&[kind]); + } } /// Count one beacon gossip verdict and how long it took from arrival. @@ -557,6 +561,49 @@ impl Drop for PoolInUse { } } +/// Gossip kinds verified in batches (see [`crate::beacon::batched`]), as the +/// `kind` label of the two batch metrics. +const BATCHED_KINDS: &[&str] = &["votes", "sync_committee", "column"]; + +/// Buckets for how many messages one drain pass verified together, up to the +/// largest batch a drainer takes. +const GOSSIP_BATCH_SIZE_BUCKETS: &[f64] = &[ + 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, +]; + +static LEAN_BEACON_GOSSIP_BATCH_SIZE: LazyLock = LazyLock::new(|| { + register_histogram_vec!( + "lean_beacon_gossip_batch_size", + "Messages verified together in one drain pass of a batched gossip kind", + &["kind"], + GOSSIP_BATCH_SIZE_BUCKETS.to_vec() + ) + .unwrap() +}); + +static LEAN_BEACON_GOSSIP_BATCH_FALLBACK_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter_vec!( + "lean_beacon_gossip_batch_fallback_total", + "Batches whose combined check failed, so each message was checked on its own", + &["kind"] + ) + .unwrap() +}); + +/// Record that a drain pass verified `size` messages together. +pub(crate) fn observe_gossip_batch_size(kind: &str, size: usize) { + LEAN_BEACON_GOSSIP_BATCH_SIZE + .with_label_values(&[kind]) + .observe(size as f64); +} + +/// Count one batch that fell back to per-message checks. +pub(crate) fn inc_gossip_batch_fallback(kind: &str) { + LEAN_BEACON_GOSSIP_BATCH_FALLBACK_TOTAL + .with_label_values(&[kind]) + .inc(); +} + /// Count one object refused for want of a permit in `pool`. pub(crate) fn inc_validation_permit_exhausted(pool: &str) { LEAN_BEACON_VALIDATION_PERMIT_EXHAUSTED_TOTAL diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 3327ee4af..76cc2b899 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -2841,6 +2841,7 @@ pub(crate) mod tests { crate::SEEN_SYNC_AGGREGATORS_CAPACITY, crate::SEEN_SYNC_DATA_CAPACITY, ), + batchers: Default::default(), sync_validation_permits: std::sync::Arc::new(tokio::sync::Semaphore::new( crate::SYNC_VALIDATION_PERMITS, )), diff --git a/crates/net/rpc/Cargo.toml b/crates/net/rpc/Cargo.toml index d07713af4..547639dbf 100644 --- a/crates/net/rpc/Cargo.toml +++ b/crates/net/rpc/Cargo.toml @@ -33,7 +33,7 @@ jemalloc_pprof.workspace = true futures-util = "0.3" [dev-dependencies] -ethlambda-validator.workspace = true +ethlambda-validator = { workspace = true, features = ["test-utils"] } spawned-concurrency.workspace = true ethlambda-state-transition = { workspace = true, features = ["test-utils"] } ethlambda-types.workspace = true diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 3b6d8d3d4..8a4105671 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -1492,6 +1492,7 @@ mod gloas { None, ), ), + ethlambda_validator::slashing_protection::SlashingProtection::in_memory(), ); let duty = ProposerDutyDto { pubkey: encode_hex(&pubkey.0), @@ -1610,6 +1611,7 @@ mod gloas { None, ), ), + ethlambda_validator::slashing_protection::SlashingProtection::in_memory(), ); let duty = ProposerDutyDto { pubkey: encode_hex(&pubkey.0), diff --git a/crates/validator/Cargo.toml b/crates/validator/Cargo.toml index c227e995f..b34e72488 100644 --- a/crates/validator/Cargo.toml +++ b/crates/validator/Cargo.toml @@ -26,10 +26,11 @@ serde_yaml_ng.workspace = true hex.workspace = true thiserror.workspace = true tracing.workspace = true -# `time` for the duty loop's sleep between slots. Declared here rather than -# relied on through workspace feature unification, so this crate still builds -# if it is ever compiled on its own. -tokio = { workspace = true, features = ["time", "macros"] } +# `time` for the duty loop's sleep between slots, `rt` for `spawn_blocking`, +# which keeps the slashing-protection record's disk flush off the executor. +# Declared here rather than relied on through workspace feature unification, +# so this crate still builds if it is ever compiled on its own. +tokio = { workspace = true, features = ["time", "macros", "rt"] } reqwest = { workspace = true, features = ["json"] } # BLS12-381. Same version and backend the state transition verifies with, so a @@ -72,3 +73,15 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util" tower = { version = "0.5", features = ["util"] } http-body-util = "0.1" tempfile = "3" + +[features] +# Exposes `SlashingProtection::in_memory` to other crates' tests. +test-utils = [] + +# Skipped by a plain `cargo test`: it reads the EIP-3076 interchange vectors, +# which `make slashing-protection-interchange-tests` downloads, and fails +# without them. `make test-node` runs it by name. +[[test]] +name = "interchange_vectors" +path = "tests/interchange_vectors.rs" +test = false diff --git a/crates/validator/src/attestation.rs b/crates/validator/src/attestation.rs index a5984e590..5372b8470 100644 --- a/crates/validator/src/attestation.rs +++ b/crates/validator/src/attestation.rs @@ -1,14 +1,14 @@ //! Producing and publishing this slot's attestations. +use std::collections::HashSet; use std::sync::Arc; use ethlambda_types::beacon::containers::shared::AttestationData; -use ethlambda_types::beacon::primitives::Slot; +use ethlambda_types::beacon::primitives::{BlsPubkey, Slot}; use ethlambda_types::beacon::signing::compute_epoch_at_slot; use tokio::sync::RwLock; use tracing::{error, info, warn}; -use crate::attestation_guard::AttestationGuard; use crate::beacon_node::dto::{ AttestationDataOutDto, AttesterDutyDto, SingleAttestationDto, encode_hex, parse_pubkey, }; @@ -16,6 +16,7 @@ use crate::beacon_node::{BeaconNodeApi, validate_attestation_data}; use crate::error::Result; use crate::keys::ValidatorStore; use crate::signing::SigningContext; +use crate::slashing_protection::SharedProtection; /// What one slot's attestation duty produced. /// @@ -29,8 +30,16 @@ use crate::signing::SigningContext; pub struct Attested { /// How many attestations the beacon node accepted. pub published: usize, - /// The data every attestation in this slot was signed over, or `None` when - /// there was no duty to fetch it for. + /// How many of the slot's duties slashing protection withheld: judged + /// unsafe to sign, or, when the record could not be written, the whole + /// batch, since nothing may be signed then. Kept apart from `published` + /// so that a slot with nothing published can be told from a slot with + /// nothing to publish, which an `Ok` with a count of zero alone cannot. + /// The cause is logged, and counted, where it is decided. + pub refused: usize, + /// The slot's attestation data, which any attestation of it is signed + /// over, or `None` when there was no duty to fetch it for. Not a record of + /// what was signed: `published` and `refused` are. /// /// An `Option` rather than a zeroed default, because a caller handed a /// default would ask for the aggregate of an attestation nobody made. It is @@ -43,25 +52,21 @@ pub struct Attested { pub struct AttestationService { beacon_node: Arc, context: Arc, - /// What this process has already signed, per validator. - /// - /// A `std::sync::Mutex` rather than a `tokio` one on purpose: the critical - /// section is a hash lookup and an insert with no `.await` inside it, so an - /// async mutex would buy nothing and cost a scheduling point. The guard is - /// taken and dropped per validator inside the signing loop, which already - /// holds the store's read lock, so it must not be the thing that blocks. - /// - /// Not slashing protection. See [`AttestationGuard`] for exactly what it - /// does and does not cover. - guard: std::sync::Mutex, + /// What every validator has signed, shared with the proposal service and + /// the keymanager. See [`crate::slashing_protection`]. + protection: SharedProtection, } impl AttestationService { - pub fn new(beacon_node: Arc, context: Arc) -> Self { + pub fn new( + beacon_node: Arc, + context: Arc, + protection: SharedProtection, + ) -> Self { Self { beacon_node, context, - guard: std::sync::Mutex::new(AttestationGuard::new()), + protection, } } @@ -80,9 +85,9 @@ impl AttestationService { /// On a submission failure every signature in the batch has already been /// produced. Calling this again re-fetches the attestation data, and if the /// head moved in between it signs a *different* message for the same - /// validator and slot. This client keeps no slashing-protection record, so - /// nothing would catch that, and two distinct attestations for one slot from - /// one validator is a slashable offence. + /// validator and slot. Slashing protection refuses it, since the first call + /// already recorded that target, so a retry here publishes nothing: the + /// attestations are lost rather than doubled. /// /// A caller that wants to retry must resubmit the batch this call already /// built, not call this again. Today's duty loop does neither: it logs the @@ -102,6 +107,7 @@ impl AttestationService { if duties.is_empty() { return Ok(Attested { published: 0, + refused: 0, data: None, }); } @@ -142,6 +148,77 @@ impl AttestationService { let data_dto = AttestationDataOutDto::from(&data); + let candidates: Vec<(&AttesterDutyDto, BlsPubkey)> = duties + .iter() + .filter_map(|duty| match parse_pubkey(&duty.pubkey) { + Ok(pubkey) => Some((duty, pubkey)), + Err(err) => { + error!(%slot, validator = duty.validator_index, %err, "Duty carried an unreadable pubkey"); + None + } + }) + .collect(); + + // Only keys this client holds. A duty can name one the keymanager + // deleted since the schedule was fetched; recording it would burn its + // watermark for nothing, or, on a persisted record, refuse it as + // unregistered and count a refusal that was never going to be a + // signature. A snapshot, so the read guard is gone before the write; + // a key deleted between it and the signing below fails there as + // unknown, and its watermark stays advanced, which only costs a key + // on its way out. + let held: HashSet = store.read().await.pubkeys().into_iter().collect(); + let candidates: Vec<(&AttesterDutyDto, BlsPubkey)> = candidates + .into_iter() + .filter(|(duty, pubkey)| { + let held = held.contains(pubkey); + if !held { + tracing::debug!( + %slot, + validator = duty.validator_index, + "Duty names a key this client no longer holds; skipped" + ); + } + held + }) + .collect(); + + if candidates.is_empty() { + // Nothing to record or sign; the data still names the aggregate. + return Ok(Attested { + published: 0, + refused: 0, + data: Some(data), + }); + } + + // Check and durably record the whole batch before any signature exists: + // a signature that exists is a signature that can escape, whatever the + // code after it intended. One write for the slot, off the executor, + // and before the store's read lock below is taken, so the disk flush + // never holds up a keymanager writer queued behind it. + let pubkeys: Vec = candidates.iter().map(|(_, pubkey)| *pubkey).collect(); + let verdicts = crate::slashing_protection::blocking(&self.protection, move |protection| { + protection.check_and_record_attestations(&pubkeys, &data) + }) + .await; + let verdicts = match verdicts { + Ok(verdicts) => verdicts, + Err(err) => { + // Nothing may be signed. Nothing was recorded either, except + // after a rename whose directory flush failed, where the target + // is on disk and in memory and a retry is refused (see + // `SlashingProtection::check_and_record_attestations`). + error!(%slot, %err, "Slashing protection could not record this slot's attestations; refusing to sign any"); + crate::metrics::inc_attestations_refused_by(candidates.len() as u64); + return Ok(Attested { + published: 0, + refused: candidates.len(), + data: Some(data), + }); + } + }; + // The read guard is scoped to this block alone and must never be widened to cover an // await. `tokio::sync::RwLock` is write-preferring, so a keymanager // import or delete queuing for the write lock blocks every read behind @@ -149,50 +226,22 @@ impl AttestationService { // import holds its own write lock for the inserts alone, doing the slow // EIP-2335 derivation outside it (see `http_api::keystores::import`), // so this is the duty path keeping its half of the same bargain. - let mut attestations = Vec::with_capacity(duties.len()); + let mut attestations = Vec::with_capacity(candidates.len()); + let mut refused = 0; { let _timing = crate::metrics::time_signing(); let store = store.read().await; - for duty in duties { - let pubkey = match parse_pubkey(&duty.pubkey) { - Ok(pubkey) => pubkey, - Err(err) => { - error!(%slot, validator = duty.validator_index, %err, "Duty carried an unreadable pubkey"); - continue; - } - }; - - // Refuse before signing, not after: a signature that exists is - // a signature that can escape, whatever the code after it - // intended. This closes the double-vote shapes reachable - // within one run (a backward clock step, a schedule replaced - // mid-epoch); it is not slashing protection and does not - // pretend to be. See `AttestationGuard`. - // - // Held per validator, and dropped before the next iteration. - // The lock is poisoned only by a panic while it is held, and - // nothing inside can panic; `unwrap` on it would still be a - // crash in the duty path, so a poisoned lock is treated as a - // refusal for every remaining validator instead. - match self.guard.lock() { - Ok(mut guard) => { - if let Err(refusal) = guard.check_and_record(&pubkey, &data) { - warn!( - %slot, - validator = duty.validator_index, - %refusal, - "Refusing to sign: this process already signed a conflicting \ - attestation for this validator" - ); - crate::metrics::inc_attestations_refused(); - continue; - } - } - Err(err) => { - error!(%slot, %err, "Attestation guard is poisoned; refusing to sign"); - crate::metrics::inc_attestations_refused(); - continue; - } + for ((duty, pubkey), verdict) in candidates.into_iter().zip(verdicts) { + if let Err(refusal) = verdict { + warn!( + %slot, + validator = duty.validator_index, + %refusal, + "Refusing to sign: slashing protection judged this attestation unsafe" + ); + crate::metrics::inc_attestations_refused(); + refused += 1; + continue; } // One validator failing must not cost the others their attestation. @@ -217,6 +266,7 @@ impl AttestationService { if attestations.is_empty() { return Ok(Attested { published: 0, + refused, data: Some(data), }); } @@ -238,6 +288,7 @@ impl AttestationService { crate::metrics::inc_attestations_published(published as u64); Ok(Attested { published, + refused, data: Some(data), }) } @@ -250,6 +301,7 @@ mod tests { // error, since `validate_attestation_data` produces them now. use crate::beacon_node::mock::MockBeaconNode; use crate::error::Error; + use crate::slashing_protection::SlashingProtection; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::shared::{AttestationData, Checkpoint}; use ethlambda_types::beacon::preset; @@ -268,8 +320,8 @@ mod tests { /// one key used twice. A pubkey resolves to exactly one validator index on /// chain, and `ValidatorStore` is keyed by pubkey so a duplicate collapses, /// so one key under two indices is a shape that cannot occur. It also now - /// trips `AttestationGuard`, correctly: two duties for one validator in one - /// epoch is the double-vote shape the guard exists to refuse. + /// trips slashing protection, correctly: two duties for one validator in + /// one epoch is the double-vote shape it exists to refuse. fn other_secret() -> [u8; 32] { let mut bytes = secret(); // Perturb a low byte: still a valid scalar, comfortably below the @@ -311,7 +363,8 @@ mod tests { .expect("inserts"); let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); - let service = AttestationService::new(node.clone(), context()); + let service = + AttestationService::new(node.clone(), context(), SlashingProtection::in_memory()); let duties = vec![ duty(&encode_hex(&first.0), 1337, 96, 3), @@ -333,7 +386,7 @@ mod tests { ); } - /// The guard engaging in the real signing path, not just in isolation. + /// Slashing protection engaging in the real signing path, not just in isolation. /// /// This is the backward-clock-step shape: the duty loop derives its slot /// from the wall clock, so an NTP correction can re-enter a slot already @@ -346,7 +399,8 @@ mod tests { let pubkey = store.insert_secret("test", &secret()).expect("inserts"); let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); - let service = AttestationService::new(node.clone(), context()); + let service = + AttestationService::new(node.clone(), context(), SlashingProtection::in_memory()); let duties = vec![duty(&encode_hex(&pubkey.0), 1337, 96, 3)]; let store = RwLock::new(store); @@ -357,8 +411,9 @@ mod tests { .await .expect("the second call is not an error, it simply signs nothing"); - assert_eq!(first.published, 1); + assert_eq!((first.published, first.refused), (1, 0)); assert_eq!(second.published, 0, "the second attempt must be refused"); + assert_eq!(second.refused, 1, "and reported as refused, not as absent"); assert_eq!( node.submitted().len(), 1, @@ -366,6 +421,90 @@ mod tests { ); } + /// A record kept in `dir`, with every key in `keys` registered. + fn persisted( + dir: &std::path::Path, + keys: &[ethlambda_types::beacon::primitives::BlsPubkey], + ) -> SharedProtection { + let protection = SlashingProtection::open(dir, Root::ZERO, keys, true).expect("opens"); + Arc::new(std::sync::Mutex::new(protection)) + } + + /// The restart shape the old in-memory guard could not cover: a fresh + /// service over the same directory still refuses the double vote, and + /// reaches the beacon node with nothing. + #[tokio::test] + async fn a_restart_does_not_forget_what_was_attested() { + let mut store = ValidatorStore::new(); + let pubkey = store.insert_secret("test", &secret()).expect("inserts"); + let store = RwLock::new(store); + let duties = vec![duty(&encode_hex(&pubkey.0), 1337, 96, 3)]; + let dir = tempfile::tempdir().expect("temp dir"); + + let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); + let first = + AttestationService::new(node.clone(), context(), persisted(dir.path(), &[pubkey])) + .attest(96, &duties, &store) + .await + .expect("attests"); + assert_eq!(first.published, 1); + + // The first service, and with it the lock on the record, is gone. + let node_again = Arc::new(MockBeaconNode::new().with_attestation_data(96)); + let second = AttestationService::new( + node_again.clone(), + context(), + persisted(dir.path(), &[pubkey]), + ) + .attest(96, &duties, &store) + .await + .expect("not an error, simply signs nothing"); + assert_eq!( + second.published, 0, + "the double vote must be refused after a restart" + ); + assert!(node_again.submitted().is_empty()); + } + + /// A record that cannot be written means nothing is signed: the write is + /// what makes a signature safe to exist. + #[cfg(unix)] + #[tokio::test] + async fn a_record_that_cannot_be_written_signs_nothing() { + use std::os::unix::fs::PermissionsExt as _; + + // SAFETY: `geteuid` reads a process attribute and cannot fail. + if unsafe { libc::geteuid() } == 0 { + eprintln!("skipped: root bypasses directory permissions"); + return; + } + let mut store = ValidatorStore::new(); + let pubkey = store.insert_secret("test", &secret()).expect("inserts"); + let store = RwLock::new(store); + let dir = tempfile::tempdir().expect("temp dir"); + let protection = persisted(dir.path(), &[pubkey]); + + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o500)) + .expect("chmod"); + let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); + let attested = AttestationService::new(node.clone(), context(), protection) + .attest(96, &[duty(&encode_hex(&pubkey.0), 1337, 96, 3)], &store) + .await; + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)) + .expect("chmod back"); + + let attested = attested.expect("not an error"); + assert_eq!(attested.published, 0); + assert_eq!( + attested.refused, 1, + "a record that could not be written refuses the whole batch, visibly" + ); + assert!( + node.submitted().is_empty(), + "nothing may reach the beacon node" + ); + } + /// The mid-epoch schedule replacement shape: the same validator moved to a /// different slot within one epoch. Both slots are in epoch 3, so the /// second is a double vote even though the slot differs. @@ -377,7 +516,8 @@ mod tests { // 96 and 101 are both in epoch 3 (96 / 32 == 101 / 32 == 3). let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); - let service = AttestationService::new(node.clone(), context()); + let service = + AttestationService::new(node.clone(), context(), SlashingProtection::in_memory()); let store = RwLock::new(store); let first = service @@ -387,12 +527,12 @@ mod tests { assert_eq!(first.published, 1); let node_again = Arc::new(MockBeaconNode::new().with_attestation_data(101)); - // Same service, so the same guard; a new mock only because the mock + // Same slashing-protection record; a new mock only because the mock // pins one slot's data at a time. let service = AttestationService { beacon_node: node_again.clone(), context: context(), - guard: service.guard, + protection: service.protection, }; let second = service .attest(101, &[duty(&pubkey_hex, 1337, 101, 3)], &store) @@ -420,7 +560,8 @@ mod tests { .expect("inserts"); let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); - let service = AttestationService::new(node.clone(), context()); + let service = + AttestationService::new(node.clone(), context(), SlashingProtection::in_memory()); let duties = vec![ duty(&encode_hex(&first.0), 1337, 96, 3), @@ -446,7 +587,8 @@ mod tests { let known = store.insert_secret("test", &secret()).expect("inserts"); let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); - let service = AttestationService::new(node.clone(), context()); + let service = + AttestationService::new(node.clone(), context(), SlashingProtection::in_memory()); let duties = vec![ duty(&encode_hex(&[0x99; 48]), 1, 96, 3), @@ -467,7 +609,8 @@ mod tests { // after the signing loop, where the loop produced nothing to send. let store = RwLock::new(ValidatorStore::new()); let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); - let service = AttestationService::new(node.clone(), context()); + let service = + AttestationService::new(node.clone(), context(), SlashingProtection::in_memory()); let duties = vec![ duty(&encode_hex(&[0x99; 48]), 1, 96, 3), @@ -494,7 +637,7 @@ mod tests { let pubkey = store.insert_secret("test", &secret()).expect("inserts"); let store = RwLock::new(store); let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); - let service = AttestationService::new(node, context()); + let service = AttestationService::new(node, context(), SlashingProtection::in_memory()); let attested = service .attest(96, &[duty(&encode_hex(&pubkey.0), 1337, 96, 3)], &store) @@ -511,7 +654,11 @@ mod tests { /// aggregate of an attestation nobody made. #[tokio::test] async fn no_duties_means_no_data_to_aggregate_against() { - let service = AttestationService::new(Arc::new(MockBeaconNode::new()), context()); + let service = AttestationService::new( + Arc::new(MockBeaconNode::new()), + context(), + SlashingProtection::in_memory(), + ); let attested = service .attest(96, &[], &RwLock::new(ValidatorStore::new())) .await @@ -522,7 +669,8 @@ mod tests { #[tokio::test] async fn no_duties_means_no_request_at_all() { let node = Arc::new(MockBeaconNode::new()); - let service = AttestationService::new(node.clone(), context()); + let service = + AttestationService::new(node.clone(), context(), SlashingProtection::in_memory()); let published = service .attest(96, &[], &RwLock::new(ValidatorStore::new())) .await @@ -534,7 +682,7 @@ mod tests { #[tokio::test] async fn a_failing_attestation_data_fetch_is_reported() { let node = Arc::new(MockBeaconNode::new()); - let service = AttestationService::new(node, context()); + let service = AttestationService::new(node, context(), SlashingProtection::in_memory()); let mut store = ValidatorStore::new(); let pubkey = store.insert_secret("test", &secret()).expect("inserts"); let store = RwLock::new(store); @@ -558,7 +706,8 @@ mod tests { let pubkey_hex = encode_hex(&pubkey.0); let node = Arc::new(MockBeaconNode::new().with_attestation_data(96)); - let service = AttestationService::new(node.clone(), context()); + let service = + AttestationService::new(node.clone(), context(), SlashingProtection::in_memory()); let duties = vec![duty(&pubkey_hex, 1337, 97, 3)]; let err = service @@ -601,7 +750,8 @@ mod tests { }, }); let node = Arc::new(node); - let service = AttestationService::new(node.clone(), context()); + let service = + AttestationService::new(node.clone(), context(), SlashingProtection::in_memory()); let duties = vec![duty(&pubkey_hex, 1337, 96, 3)]; let err = service @@ -632,24 +782,32 @@ mod tests { let first_altair_slot = altair_fork_epoch * preset::SLOTS_PER_EPOCH; let phase0_node = Arc::new(MockBeaconNode::new().with_attestation_data(last_phase0_slot)); - AttestationService::new(phase0_node.clone(), context()) - .attest( - last_phase0_slot, - &[duty(&pubkey_hex, 1337, last_phase0_slot, 3)], - &store, - ) - .await - .expect("attests"); + AttestationService::new( + phase0_node.clone(), + context(), + SlashingProtection::in_memory(), + ) + .attest( + last_phase0_slot, + &[duty(&pubkey_hex, 1337, last_phase0_slot, 3)], + &store, + ) + .await + .expect("attests"); let altair_node = Arc::new(MockBeaconNode::new().with_attestation_data(first_altair_slot)); - AttestationService::new(altair_node.clone(), context()) - .attest( - first_altair_slot, - &[duty(&pubkey_hex, 1337, first_altair_slot, 3)], - &store, - ) - .await - .expect("attests"); + AttestationService::new( + altair_node.clone(), + context(), + SlashingProtection::in_memory(), + ) + .attest( + first_altair_slot, + &[duty(&pubkey_hex, 1337, first_altair_slot, 3)], + &store, + ) + .await + .expect("attests"); assert_eq!( phase0_node.last_submitted_fork_name().as_deref(), @@ -688,10 +846,14 @@ mod tests { .with_attestation_data(slot) .with_attestation_index(1), ); - let attested = AttestationService::new(node.clone(), gloas_context()) - .attest(slot, &[duty(&pubkey_hex, 1337, slot, 3)], &store) - .await - .expect("attests"); + let attested = AttestationService::new( + node.clone(), + gloas_context(), + SlashingProtection::in_memory(), + ) + .attest(slot, &[duty(&pubkey_hex, 1337, slot, 3)], &store) + .await + .expect("attests"); assert_eq!(attested.data.expect("data").index, 1); let submitted = node.submitted(); @@ -744,10 +906,14 @@ mod tests { let slot = 9 * preset::SLOTS_PER_EPOCH + 3; let node = Arc::new(MockBeaconNode::new().with_attestation_data(slot)); - AttestationService::new(node.clone(), gloas_context()) - .attest(slot, &[duty(&pubkey_hex, 1337, slot, 3)], &store) - .await - .expect("attests"); + AttestationService::new( + node.clone(), + gloas_context(), + SlashingProtection::in_memory(), + ) + .attest(slot, &[duty(&pubkey_hex, 1337, slot, 3)], &store) + .await + .expect("attests"); assert_eq!(node.attestation_data_forks(), vec![ForkName::Phase0]); } } diff --git a/crates/validator/src/attestation_guard.rs b/crates/validator/src/attestation_guard.rs deleted file mode 100644 index 5a5ebc3e4..000000000 --- a/crates/validator/src/attestation_guard.rs +++ /dev/null @@ -1,377 +0,0 @@ -//! An in-process guard against signing two conflicting attestations. -//! -//! # What this is not -//! -//! **This is not slashing protection.** It holds no history on disk, so it -//! knows nothing about what a previous run of this process signed, and nothing -//! about what another process holding the same keys is signing right now. A -//! restart empties it. Its records are gone the moment the process is. -//! -//! This client keeps no signing history by design (see the crate -//! documentation). That decision is not reversed here and this module is not a -//! step toward reversing it: durable slashing protection is a different thing, -//! with a durability requirement this deliberately does not have. -//! -//! # What it is -//! -//! A cheap check that closes the double-vote shapes this client can reach -//! *within one run*, which were otherwise reachable through ordinary -//! operation rather than through operator error: -//! -//! - **A backward wall-clock step.** The duty loop derives its slot from -//! `SystemTime::now()`, which is not monotonic. An NTP correction of a few -//! seconds re-enters a slot already attested, and the re-fetched attestation -//! data can differ if a block arrived in between. Two different attestations -//! under one target epoch is a double vote. -//! - **A schedule replaced mid-epoch.** A duty refresh that fails at an epoch -//! boundary and succeeds a few slots later can move a validator to a -//! different slot in the *same* epoch, and the loop then attests a second -//! time under that target. -//! -//! Neither needs a hostile beacon node or a second instance. Both are ordinary -//! things that happen to running systems. -//! -//! # The rule -//! -//! EIP-3076's minimal variant, per validator: remember the highest source and -//! target epoch signed, and refuse anything that does not strictly advance the -//! target or that would reach back below the source. -//! -//! That is deliberately more conservative than the full slashing conditions, -//! which would need the whole history to evaluate. With one record per -//! validator it cannot distinguish "surrounds an attestation from six epochs -//! ago" from "surrounds nothing"; requiring monotonic progress makes the -//! question unnecessary. An honest validator's attestations already advance -//! this way, so the conservatism costs nothing in normal operation. -//! -//! Refusing is always safe here: a skipped attestation is a missed reward, and -//! a double vote is a slashing. - -use std::collections::HashMap; - -use ethlambda_types::beacon::containers::shared::AttestationData; -use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch}; - -/// The highest source and target epoch signed for one validator. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct Signed { - source: Epoch, - target: Epoch, -} - -/// Why an attestation was refused. -/// -/// Carried out of [`AttestationGuard::check`] so the caller can log which rule -/// fired rather than a bare "refused". The two are different operational -/// situations: a repeated target usually means the clock moved or a schedule -/// was replaced, while a regressed source means something stranger and is -/// worth looking at. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Refusal { - /// The target epoch is at or below one already signed for this validator. - /// - /// The double-vote shape. Equality is refused as well as regression: the - /// same target signed twice is a double vote unless the two attestations - /// are byte-identical, and this guard deliberately does not keep enough to - /// tell those apart. Re-signing identical data would be harmless but is - /// also pointless, since the client does not retry within a slot. - TargetNotAdvanced { signed: Epoch, proposed: Epoch }, - /// The source epoch is below one already signed for this validator. - /// - /// The surround shape: a lower source with a higher target surrounds the - /// earlier attestation. - SourceRegressed { signed: Epoch, proposed: Epoch }, -} - -impl std::fmt::Display for Refusal { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::TargetNotAdvanced { signed, proposed } => write!( - f, - "target epoch {proposed} does not advance past {signed}, already signed this run" - ), - Self::SourceRegressed { signed, proposed } => write!( - f, - "source epoch {proposed} is below {signed}, already signed this run" - ), - } - } -} - -/// Per-validator record of the highest attestation signed in this run. -/// -/// Not `Clone`: one guard per process is the point. A copy would be a second -/// opinion about what has been signed, which is worse than no opinion. -#[derive(Debug, Default)] -pub struct AttestationGuard { - signed: HashMap, -} - -impl AttestationGuard { - pub fn new() -> Self { - Self::default() - } - - /// Whether `data` may be signed for `pubkey`, without recording anything. - /// - /// Separate from [`Self::record`] so a caller can decide before paying for - /// a signature, and so this stays a pure function over the guard's state - /// and therefore directly testable. - pub fn check(&self, pubkey: &BlsPubkey, data: &AttestationData) -> Result<(), Refusal> { - let Some(signed) = self.signed.get(pubkey) else { - // Nothing signed for this validator this run. Note what this does - // *not* mean: a previous run may have signed anything at all. That - // gap is the crate-level scope decision, not an oversight here. - return Ok(()); - }; - - if data.target.epoch <= signed.target { - return Err(Refusal::TargetNotAdvanced { - signed: signed.target, - proposed: data.target.epoch, - }); - } - if data.source.epoch < signed.source { - return Err(Refusal::SourceRegressed { - signed: signed.source, - proposed: data.source.epoch, - }); - } - Ok(()) - } - - /// Record that `data` was signed for `pubkey`. - /// - /// Takes the maximum of each epoch rather than overwriting, so that a - /// caller recording out of order cannot lower the bar. Nothing does that - /// today; the guard is cheap to make order-independent and expensive to - /// debug if it ever is not. - pub fn record(&mut self, pubkey: BlsPubkey, data: &AttestationData) { - let entry = self.signed.entry(pubkey).or_insert(Signed { - source: data.source.epoch, - target: data.target.epoch, - }); - entry.source = entry.source.max(data.source.epoch); - entry.target = entry.target.max(data.target.epoch); - } - - /// Check and record in one step, for the common call site. - /// - /// Recording only on success is what makes a refusal idempotent: a - /// refused attestation must not move the bar it was measured against. - pub fn check_and_record( - &mut self, - pubkey: &BlsPubkey, - data: &AttestationData, - ) -> Result<(), Refusal> { - self.check(pubkey, data)?; - self.record(*pubkey, data); - Ok(()) - } - - /// How many validators have signed something this run. For metrics. - pub fn len(&self) -> usize { - self.signed.len() - } - - pub fn is_empty(&self) -> bool { - self.signed.is_empty() - } -} - -#[cfg(test)] -mod tests { - use ethlambda_types::beacon::containers::shared::Checkpoint; - use ethlambda_types::beacon::primitives::{H256, Root}; - - use super::*; - - fn pubkey(byte: u8) -> BlsPubkey { - BlsPubkey([byte; 48]) - } - - /// An attestation at `slot` voting source `source` and target `target`. - /// - /// `beacon_block_root` is a parameter because the double-vote case that - /// matters most is two attestations with one target epoch and *different* - /// roots, which is what a re-fetch after a block arrival produces. - fn data(slot: u64, source: Epoch, target: Epoch, root: u8) -> AttestationData { - AttestationData { - slot, - index: 0, - beacon_block_root: H256([root; 32]), - source: Checkpoint { - epoch: source, - root: Root::ZERO, - }, - target: Checkpoint { - epoch: target, - root: Root::ZERO, - }, - } - } - - #[test] - fn a_first_attestation_is_allowed() { - let mut guard = AttestationGuard::new(); - assert!( - guard - .check_and_record(&pubkey(1), &data(96, 2, 3, 0xaa)) - .is_ok() - ); - } - - #[test] - fn advancing_the_target_is_allowed() { - let mut guard = AttestationGuard::new(); - guard - .check_and_record(&pubkey(1), &data(96, 2, 3, 0xaa)) - .expect("first"); - - assert!( - guard - .check_and_record(&pubkey(1), &data(128, 3, 4, 0xbb)) - .is_ok() - ); - } - - /// The backward-clock-step case. Same validator, same target epoch, a - /// different block root because the data was re-fetched after a block - /// arrived. This is the double vote the guard exists to stop. - #[test] - fn re_attesting_one_target_with_different_data_is_refused() { - let mut guard = AttestationGuard::new(); - guard - .check_and_record(&pubkey(1), &data(96, 2, 3, 0xaa)) - .expect("first"); - - let err = guard - .check_and_record(&pubkey(1), &data(96, 2, 3, 0xbb)) - .expect_err("a second vote for one target must be refused"); - - assert_eq!( - err, - Refusal::TargetNotAdvanced { - signed: 3, - proposed: 3 - } - ); - } - - /// The mid-epoch schedule replacement case: a different *slot* in the same - /// epoch, which is still one target epoch and still a double vote. - #[test] - fn a_different_slot_in_the_same_target_epoch_is_refused() { - let mut guard = AttestationGuard::new(); - guard - .check_and_record(&pubkey(1), &data(96, 2, 3, 0xaa)) - .expect("first"); - - assert!( - guard - .check_and_record(&pubkey(1), &data(101, 2, 3, 0xbb)) - .is_err(), - "a second slot under one target epoch is a double vote" - ); - } - - #[test] - fn a_regressed_target_is_refused() { - let mut guard = AttestationGuard::new(); - guard - .check_and_record(&pubkey(1), &data(128, 3, 4, 0xaa)) - .expect("first"); - - assert!( - guard - .check_and_record(&pubkey(1), &data(96, 2, 3, 0xbb)) - .is_err() - ); - } - - /// The surround shape: a later target with an earlier source surrounds the - /// attestation already signed. - #[test] - fn a_regressed_source_under_an_advancing_target_is_refused() { - let mut guard = AttestationGuard::new(); - guard - .check_and_record(&pubkey(1), &data(128, 3, 4, 0xaa)) - .expect("first"); - - let err = guard - .check_and_record(&pubkey(1), &data(160, 1, 5, 0xbb)) - .expect_err("a surrounding vote must be refused"); - - assert_eq!( - err, - Refusal::SourceRegressed { - signed: 3, - proposed: 1 - } - ); - } - - /// The property that makes this per-validator rather than per-slot: two - /// validators in one epoch attest at different slots, and the second must - /// not be blocked by the first. - #[test] - fn one_validator_does_not_block_another_in_the_same_epoch() { - let mut guard = AttestationGuard::new(); - guard - .check_and_record(&pubkey(1), &data(96, 2, 3, 0xaa)) - .expect("first validator"); - - assert!( - guard - .check_and_record(&pubkey(2), &data(101, 2, 3, 0xaa)) - .is_ok(), - "a second validator attesting later in the same epoch must be allowed" - ); - } - - /// A refusal must not move the bar, or one refused attestation would - /// change what the next check compares against. - #[test] - fn a_refusal_records_nothing() { - let mut guard = AttestationGuard::new(); - guard - .check_and_record(&pubkey(1), &data(128, 3, 4, 0xaa)) - .expect("first"); - - guard - .check_and_record(&pubkey(1), &data(160, 1, 5, 0xbb)) - .expect_err("surround refused"); - - // The recorded source must still be 3, not 1: a valid attestation - // advancing from the real record is still accepted. - assert!( - guard - .check_and_record(&pubkey(1), &data(160, 3, 5, 0xcc)) - .is_ok(), - "the refused attempt must not have lowered the recorded source" - ); - } - - #[test] - fn check_alone_does_not_record() { - let guard = AttestationGuard::new(); - let data = data(96, 2, 3, 0xaa); - - guard.check(&pubkey(1), &data).expect("allowed"); - guard - .check(&pubkey(1), &data) - .expect("still allowed, because check records nothing"); - } - - #[test] - fn recording_out_of_order_does_not_lower_the_bar() { - let mut guard = AttestationGuard::new(); - guard.record(pubkey(1), &data(128, 3, 4, 0xaa)); - guard.record(pubkey(1), &data(96, 2, 3, 0xbb)); - - assert!( - guard.check(&pubkey(1), &data(101, 2, 3, 0xcc)).is_err(), - "the earlier record must not have displaced the later one" - ); - } -} diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 8251f9559..f05107ab0 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -606,8 +606,8 @@ mod tests { /// The proposal-shaped version of the stale-head finding above, and the /// more expensive one to get wrong: a block signed against a stale node's - /// answer is a block for the wrong slot, and it burns the proposal guard's - /// record for that validator on the way out. + /// answer is a block for the wrong slot, and it burns slashing + /// protection's record for that validator on the way out. #[tokio::test] async fn a_node_producing_a_block_for_the_wrong_slot_is_failed_over_from() { let stale = MockBeaconNode::new().with_block(90, 7); @@ -622,8 +622,8 @@ mod tests { } /// A node on a different fork computes a different proposer. Signing its - /// block would produce a signature the network discards, while the guard - /// records the slot as proposed and refuses the real duty. + /// block would produce a signature the network discards, while slashing + /// protection records the slot as proposed and refuses the real duty. #[tokio::test] async fn a_node_naming_the_wrong_proposer_is_failed_over_from() { let wrong = MockBeaconNode::new().with_block(96, 11); diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index c689d2a1d..49c1c54dd 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -62,6 +62,18 @@ pub fn validate_attestation_data(slot: Slot, data: &AttestationData) -> Result<( data.target.epoch ))); } + // Signing a source at or past its target would record a source the + // chain's justified checkpoint cannot have reached, and the record keeps + // it: every honest attestation afterwards would be refused as a source + // regression until justification caught up, or forever during + // non-finality. + if crate::slashing_protection::is_impossible_source(data.source.epoch, data.target.epoch) { + return Err(crate::error::Error::InconsistentResponse(format!( + "attestation data for slot {slot} carries source epoch {} at or past its target \ + epoch {}", + data.source.epoch, data.target.epoch + ))); + } Ok(()) } @@ -156,11 +168,11 @@ pub enum Published { /// stuck on a stale head is never failed over from. /// /// Both fields are checked because both end up signed. The slot selects the -/// signing domain and is the whole of what the proposal guard keys on. The +/// signing domain and is the whole of what slashing protection keys a block on. The /// proposer index is not something this client chooses either: a node on a /// different fork computes a different proposer, and a block naming someone /// else is one this client's key can only sign uselessly, while still burning -/// the guard's record for that slot. +/// the slot in slashing protection's record. pub fn validate_produced_block(request: &BlockRequest, block: &ProducedBlock) -> Result<()> { if block.slot() != request.slot { return Err(crate::error::Error::InconsistentResponse(format!( @@ -322,9 +334,10 @@ pub trait BeaconNodeApi: Send + Sync { /// /// An implementation must reject data whose `slot` is not the one asked /// for, and whose `target.epoch` is not that slot's epoch. Both are - /// signable material, and this client keeps no slashing-protection record, + /// signable material, and slashing protection records whatever it allows, /// so a node answering about the wrong slot must never reach the signing - /// path. + /// path: a target from the future would raise the validator's watermark + /// and cost it every attestation until the chain caught up. /// /// It matters that this is the *implementation's* job rather than the /// caller's, because of where failover sits. @@ -571,3 +584,46 @@ pub fn validate_sync_contribution( } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use ethlambda_types::beacon::containers::shared::Checkpoint; + + fn data(slot: Slot, source: Epoch, target: Epoch) -> AttestationData { + AttestationData { + slot, + index: 0, + beacon_block_root: Root::ZERO, + source: Checkpoint { + epoch: source, + root: Root::ZERO, + }, + target: Checkpoint { + epoch: target, + root: Root::ZERO, + }, + } + } + + /// Slot 100 is in epoch 3 on the mainnet preset. + #[test] + fn attestation_data_with_a_source_at_or_past_its_target_is_refused() { + assert!(validate_attestation_data(100, &data(100, 2, 3)).is_ok()); + assert!(validate_attestation_data(100, &data(100, 0, 3)).is_ok()); + let err = validate_attestation_data(100, &data(100, 3, 3)).expect_err("source == target"); + assert!(err.to_string().contains("source epoch 3"), "{err}"); + assert!(validate_attestation_data(100, &data(100, 4, 3)).is_err()); + } + + /// At genesis both checkpoints are epoch 0, and that is the one valid + /// shape where the source does not precede the target. + #[test] + fn genesis_attestation_data_is_accepted() { + assert!(validate_attestation_data(0, &data(0, 0, 0)).is_ok()); + assert!( + validate_attestation_data(0, &data(0, 1, 0)).is_err(), + "a source past a genesis target is not the genesis shape" + ); + } +} diff --git a/crates/validator/src/error.rs b/crates/validator/src/error.rs index 55b82a0ff..ae65aee92 100644 --- a/crates/validator/src/error.rs +++ b/crates/validator/src/error.rs @@ -72,13 +72,15 @@ pub enum Error { /// abandoned attestation itself is never retried, by design. #[error("attestation duty for slot {slot} ran past the end of its slot")] AttestationDeadline { slot: u64 }, - /// The proposal guard refused to sign a block for this slot. + /// Slashing protection refused to sign a block for this slot, or could not + /// record it. /// - /// Not a beacon-node failure and not retryable: the whole point of a - /// refusal is that trying again produces the same answer. It is an error - /// rather than a quiet `Ok` because a refusal means the duty loop asked for - /// something it should not have, which is worth surfacing even though the - /// block was correctly suppressed. + /// Not a beacon-node failure and not retryable within the slot: a refusal + /// gives the same answer again, and a record that could not be written + /// needs an operator, not a retry. It is an error rather than a quiet `Ok` + /// because either way the duty loop asked for something that was not + /// signed, which is worth surfacing even though the block was correctly + /// suppressed. #[error("refused to propose a block for slot {slot}: {reason}")] ProposalRefused { slot: u64, reason: String }, #[error("beacon node returned an inconsistent response: {0}")] @@ -104,12 +106,46 @@ pub enum Error { #[error("signing failed for {pubkey:?}: {reason}")] Signing { pubkey: BlsPubkey, reason: String }, + /// The slashing-protection record could not be opened, read or written, + /// or refuses to start: a key with no record, another chain's record, + /// another process holding it. + /// + /// Never retryable: every one of these means nothing may be signed until + /// an operator looks. + #[error("slashing protection {path}: {reason}")] + SlashingProtection { path: String, reason: String }, + /// The record cannot be reached at all, as opposed to a record that + /// answered: its lock is poisoned because an earlier access panicked + /// while holding it, or the runtime is shutting down under the blocking + /// task. + /// + /// Kept apart from `SlashingProtection` and `Io` because the remedy + /// differs: those name a file to look at, this one names a process to + /// restart. A poisoned lock is a bug, never a disk, and it stays poisoned + /// for the life of the process, so every duty is refused until it is + /// restarted. Never retryable. + #[error("slashing protection is unusable: {reason}")] + SlashingProtectionUnusable { reason: String }, + #[error("{path}: {source}")] Io { path: String, #[source] source: std::io::Error, }, + + /// A durable write that replaced its file but could not flush the + /// directory, so the new file is what a reader sees now and may not be + /// what survives a power loss (see `secure_fs::DurableWriteError`). + #[error( + "{path}: written, but the directory could not be flushed, so the write may not survive a \ + power loss: {source}" + )] + Unflushed { + path: String, + #[source] + source: std::io::Error, + }, } impl Error { @@ -128,15 +164,18 @@ impl Error { | Self::InconsistentResponse(_) | Self::AttestationDeadline { .. } | Self::Decode(_) => true, - // A refusal is deterministic: the guard will refuse the same slot - // again, so retrying only repeats it. + // A refusal repeats itself, and a record that could not be written + // needs an operator: retrying within the slot helps neither. Self::ProposalRefused { .. } | Self::Keystore { .. } | Self::KeystoreVersion(_) | Self::KeystoreBadPassword | Self::UnknownValidator(_) | Self::Signing { .. } - | Self::Io { .. } => false, + | Self::SlashingProtection { .. } + | Self::SlashingProtectionUnusable { .. } + | Self::Io { .. } + | Self::Unflushed { .. } => false, } } } diff --git a/crates/validator/src/http_api/keystores.rs b/crates/validator/src/http_api/keystores.rs index 2232679b3..c8356decb 100644 --- a/crates/validator/src/http_api/keystores.rs +++ b/crates/validator/src/http_api/keystores.rs @@ -1,24 +1,43 @@ //! `GET`, `POST` and `DELETE /eth/v1/keystores`. //! -//! # Deviation +//! # Slashing protection //! -//! This client keeps no slashing-protection record, so the interchange data the -//! specification requires cannot be produced. `DELETE` declares -//! `slashing_protection` a required response field, so an empty but well-formed -//! EIP-3076 interchange is returned; `POST` accepts the optional -//! `slashing_protection` field and ignores it. A caller migrating keys between -//! clients must not rely on this client to carry that history. +//! The specification lets these routes carry a validator's EIP-3076 history in +//! and out, and both do: //! -//! The empty interchange [`DELETE`] returns is deliberately unusable as one: -//! its `genesis_validators_root` is all-zero rather than the chain's real -//! value. A real root would make the file look like legitimate, if empty, -//! history, which is worse than what it actually is (no history at all, -//! because this client never recorded any). The all-zero root is chosen so a -//! conformant importer rejects the file outright instead of trusting it, on -//! the theory that a loud, obvious failure at import time beats a tool -//! silently believing a validator has never signed. - -use std::path::PathBuf; +//! - `POST` judges and writes the request's `slashing_protection` before +//! touching any key: one that does not parse, or is for another chain, +//! fails every key in the request and imports none of them. Written first, +//! rather than with the keys' registration, because a key that was deleted +//! here earlier keeps its record: were the keys saved first, a crash before +//! the history landed would load such a key at its next start on the +//! watermarks it had before it left, with whatever it signed elsewhere +//! unknown. The history only ever raises watermarks, so writing it for keys +//! that then fail to save leaves nothing dangerous behind, and it is +//! written whole, entries for keys of the request that failed included: a +//! later import of such a key merges with it. The keys are registered after +//! they are saved, in one write, before any is loaded, and a registration +//! that fails takes the keys off disk again. A key with no history at all is +//! registered as never having signed, with a warning. The history may also +//! name validators that are not in the request: they get records too, and +//! a later `DELETE` of one answers `not_active`. One of them may be a key +//! this client is running now; its watermarks are raised like any other, +//! since a history saying it signed elsewhere is exactly what the record +//! must know, and the raise is logged so the operator finds out that the +//! key ran in two places. +//! - `DELETE` returns the deleted keys' records as a minimal interchange, +//! taken after the keys are out of the store, so it covers every signature +//! this process made with them. A key that is gone but still has a record is +//! reported `not_active`, and its record is kept, so importing it back later +//! starts from what this client signed with it (not from what the key may +//! have signed elsewhere in between: see `docs/spec_deviations.md` on +//! doppelganger protection). +//! +//! The history is this client's EIP-3076 minimal watermarks (see +//! [`crate::slashing_protection`]), so an export lists, per validator, its +//! highest slot and its highest source and target rather than every message. + +use std::path::{Path, PathBuf}; use std::sync::Arc; use axum::{Json, extract::State}; @@ -35,9 +54,6 @@ use crate::keys::keystore::Keystore; use crate::keys::{ValidatorDefinition, ValidatorDefinitions, ValidatorStore}; use crate::secure_fs; -/// The EIP-3076 interchange version this client reports when it has no history. -const INTERCHANGE_VERSION: &str = "5"; - /// The most keystores accepted in a single import request. /// /// Each one costs a deliberately slow EIP-2335 key derivation (scrypt at the @@ -72,7 +88,8 @@ pub struct ImportRequest { /// memory until decryption runs, and a plain `Vec` would leave /// those bytes sitting in the allocator's freed memory afterwards. pub passwords: Zeroizing>, - /// Accepted and ignored: this client keeps no slashing-protection record. + /// An EIP-3076 interchange, written before any key is saved; see the + /// module documentation. #[serde(default)] pub slashing_protection: Option, } @@ -120,8 +137,8 @@ pub struct DeleteRequest { #[derive(Debug, Serialize)] pub struct DeleteResponse { pub data: Vec, - /// Required by the specification even though this client has no history to - /// report. See the module documentation. + /// The deleted keys' records, as a minimal EIP-3076 interchange. See the + /// module documentation. pub slashing_protection: String, } @@ -142,12 +159,8 @@ pub async fn list(State(context): State) -> Json, - Json(request): Json, + Json(mut request): Json, ) -> Json { - if request.slashing_protection.is_some() { - warn!("Ignoring slashing_protection on import: this client keeps no signing history"); - } - if request.keystores.len() > MAX_KEYSTORES_PER_IMPORT { let message = format!( "request carries {} keystores, more than the limit of {MAX_KEYSTORES_PER_IMPORT}", @@ -165,6 +178,76 @@ pub async fn import( return Json(StatusResponse { data }); } + // The history is parsed once, off every lock, and written before any key + // is touched, so a request whose history is refused imports no key at + // all, and no key can be saved ahead of the history it came with (see the + // module documentation for the crash that ordering closes). Parsed on the + // blocking pool: a complete-format interchange runs to tens of megabytes + // (see `KEYMANAGER_BODY_LIMIT`), and the executor thread this handler + // runs on also drives the duty services. + let history = match request.slashing_protection.take() { + None => None, + Some(interchange) => { + let staged = tokio::task::spawn_blocking(move || { + crate::slashing_protection::SlashingProtection::stage(&interchange) + }) + .await + .unwrap_or_else(|err| { + Err(Error::SlashingProtection { + path: "interchange".to_string(), + reason: format!("could not be parsed: {err}"), + }) + }); + let written = match staged { + Ok(staged) => { + let staged = Arc::new(staged); + let absorbed = staged.clone(); + crate::slashing_protection::blocking(&context.protection, move |protection| { + protection.absorb(Some(&absorbed), &[]) + }) + .await + .map(|absorbed| (staged, absorbed)) + } + Err(err) => Err(err), + }; + match written { + Ok((staged, absorbed)) => { + info!( + validators = absorbed.named, + "Imported slashing-protection history" + ); + // A key running here whose watermarks this history raised + // signed elsewhere while it ran here: worth knowing. + let store = context.store.read().await; + for pubkey in absorbed.raised.iter().filter(|key| store.contains(key)) { + warn!( + pubkey = %encode_hex(&pubkey.0), + "Imported history raised the slashing-protection watermarks of a \ + validator this client is running; it has signed elsewhere too" + ); + } + drop(store); + Some(staged) + } + // `Error::Unflushed` is refused here too, unlike at the keys' + // registration below: the history is on disk and in memory, + // but were it lost to a power loss, a key saved on the + // strength of it would load at the next start on the record + // it had before, which is the case writing it first closes. + Err(err) => { + let message = format!("slashing_protection was not imported: {err}"); + warn!(%err, "Refusing an import whose slashing-protection history was refused"); + let data = request + .keystores + .iter() + .map(|_| KeyStatus::error(&message)) + .collect(); + return Json(StatusResponse { data }); + } + } + } + }; + // Held across this whole function: the entire open-mutate-save cycle // over the definitions file, including the slow decrypt-and-persist loop // below. This is what actually serializes concurrent import/delete @@ -194,6 +277,21 @@ pub async fn import( } }; + // The entries the file held before this request touches it, by key. A + // failed registration below removes what the request added and puts back + // what it replaced: a disabled entry, which was never loaded and so may + // have no record, must not be left enabled on disk, or the next start + // refuses the whole client for it. + let before: std::collections::HashMap = definitions + .0 + .iter() + .filter_map(|definition| { + parse_pubkey(&definition.voting_public_key) + .ok() + .map(|pubkey| (pubkey, definition.clone())) + }) + .collect(); + // Decrypt (deliberately slow: EIP-2335 key derivation) and persist every // key to disk before ever touching the store's write lock. The duty // loop's `attest` only ever holds a read lock across a short, synchronous @@ -209,32 +307,248 @@ pub async fn import( prepared.push(prepare_import(&context, &mut definitions, json, password)); } - // The definitions file is fully written by this point; nothing below - // touches it, so the guard can be released before the store lock is - // taken rather than held across it too. - drop(_definitions_guard); + // Every persisted key is registered in one durable write, before any of + // them is loaded: the duty path must never meet a key with no record. + // After persisting rather than before, so a key whose persist failed + // leaves no record behind to let a hand-copied keystore past the startup + // check later. The history goes along again so that a key it covers is + // left on it rather than warned about as having brought none; it was + // written above, so this costs nothing unless a key is new. Still under + // the definitions guard, so a concurrent delete cannot interleave. + let persisted: Vec = prepared + .iter() + .filter_map(|result| result.as_ref().ok().map(|(pubkey, _, _)| *pubkey)) + .collect(); + // Nothing to register for a request that saved no key. + let registration = if persisted.is_empty() { + Ok(crate::slashing_protection::Absorbed::default()) + } else { + let registering = persisted.clone(); + let history = history.clone(); + crate::slashing_protection::blocking(&context.protection, move |protection| { + protection.absorb(history.as_deref(), ®istering) + }) + .await + }; + // The records are on disk and in memory (see `SlashingProtection::commit`), + // only their survival of a power loss is in doubt, and a registration lost + // that way is refused at the next start: closed. So the keys are loaded, + // rather than rolled back to leave records behind for keys never imported. + let registration = match registration { + Err(Error::Unflushed { path, source }) => { + warn!( + %path, + %source, + "Slashing-protection records written, but the directory could not be flushed; \ + the imported keys are registered" + ); + Ok(crate::slashing_protection::Absorbed::default()) + } + other => other, + }; + if let Ok(absorbed) = ®istration + && absorbed.registered > 0 + { + warn!( + registered = absorbed.registered, + "Imported validator keys with no slashing-protection history; each starts as \ + never having signed" + ); + } + // A key on disk with no record is refused at the next start, and + // `SlashingProtection::open` then refuses the whole client, not the one + // key. A crash between persisting and registering leaves exactly that, + // which is the closed side to fail on. Here the write is known to have + // failed, so the keys this request added are taken off disk again, the + // entries it replaced are put back as they were, and the failure costs + // this request only. The keystore and password files go back to what was + // at their paths before, as a pair, or are removed where there was + // nothing. + let rolled_back = match ®istration { + Ok(_) => Ok(()), + Err(_) => { + let mut added = Vec::new(); + let mut replaced = Vec::new(); + for pubkey in &persisted { + match before.get(pubkey) { + Some(previous) => replaced.push((*pubkey, previous.clone())), + None => added.push(*pubkey), + } + } + let files: Vec<(BlsPubkey, &Overwritten)> = prepared + .iter() + .filter_map(|result| { + result + .as_ref() + .ok() + .map(|(pubkey, _, overwritten)| (*pubkey, overwritten)) + }) + .collect(); + undo_import(&context, &mut definitions, &added, &replaced, &files) + } + }; + if let Err(err) = &rolled_back { + tracing::error!( + %err, + "Could not undo a failed import; the next start refuses its keys until they have a \ + slashing-protection record" + ); + } + + // The definitions guard stays held across the store lock, in the order + // `delete` takes the two. Released before it, a delete of the same key + // could run in between: it would find the key on disk but not in memory, + // remove it and answer `deleted`, and the insert below would then load a + // key the operator was told is gone. let mut store = context.store.write().await; let data = prepared .into_iter() - .map(|result| match result { - Ok(secret) => finish_import(&mut store, &secret), - Err(status) => status, + .map(|result| match (result, ®istration) { + (Ok((_, secret, _)), Ok(_)) => finish_import(&mut store, &secret), + (Ok((pubkey, _, _)), Err(err)) => { + if before.contains_key(&pubkey) { + // Its keystore was rewritten; its entry is back as it was + // and nothing about it changed in memory, so it stays + // whatever it was, loaded since startup in the ordinary + // case. + let entry = if rolled_back.is_ok() { + "its definitions entry is back as it was" + } else { + "its definitions entry could not be put back" + }; + KeyStatus::error(format!( + "keystore rewritten, but the slashing-protection record could not be \ + written: {err}; {entry}" + )) + } else if rolled_back.is_err() { + // Still on disk, not loaded now, and refused at the next + // start until it has a record. + KeyStatus::error(format!( + "saved, but not loaded: its slashing-protection record could not be \ + written: {err}" + )) + } else { + KeyStatus::error(format!( + "not imported: its slashing-protection record could not be written: {err}" + )) + } + } + (Err(status), _) => status, }) .collect(); drop(store); + drop(_definitions_guard); Json(StatusResponse { data }) } +/// Undo what a failed import did: the definitions entries it added are +/// removed and the ones it replaced put back, in one save, then every +/// keystore and password file it wrote goes back to what was at its path +/// before, as a pair, or is removed where there was nothing. The file goes +/// first, since it is what the next start loads from, so a crash in between +/// leaves orphan files and never a key that would be loaded without a +/// record. A file that is already gone is not an error. +fn undo_import( + context: &KeymanagerContext, + definitions: &mut ValidatorDefinitions, + added: &[BlsPubkey], + replaced: &[(BlsPubkey, ValidatorDefinition)], + files: &[(BlsPubkey, &Overwritten)], +) -> crate::Result<()> { + if added.is_empty() && replaced.is_empty() { + return Ok(()); + } + definitions.0.retain(|definition| { + parse_pubkey(&definition.voting_public_key) + .map(|parsed| !added.contains(&parsed)) + .unwrap_or(true) + }); + for definition in &mut definitions.0 { + if let Some((_, previous)) = replaced.iter().find(|(pubkey, _)| { + parse_pubkey(&definition.voting_public_key).is_ok_and(|parsed| parsed == *pubkey) + }) { + *definition = previous.clone(); + } + } + save_definitions(context, definitions)?; + // Newest first: a key listed twice in one request snapshotted, the second + // time, what the first time wrote, so the earliest snapshot must be the + // one that lands last. + for (pubkey, overwritten) in files.iter().rev() { + let [keystore, password] = written_paths(context, pubkey)?; + put_back(&keystore, overwritten.keystore.as_deref())?; + put_back( + &password, + overwritten.password.as_deref().map(|bytes| &bytes[..]), + )?; + } + Ok(()) +} + +/// Put `path` back to `previous`: its former bytes, or nothing. +fn put_back(path: &Path, previous: Option<&[u8]>) -> crate::Result<()> { + match previous { + Some(bytes) => { + secure_fs::write_private_no_symlink(path, bytes).map_err(|source| Error::Io { + path: path.display().to_string(), + source, + }) + } + None => remove_if_present(path), + } +} + +/// Save the definitions file, keeping `definitions_unflushed` true from an +/// after-rename flush failure until a later save flushes the directory, which +/// makes every earlier rename in it durable too. +fn save_definitions( + context: &KeymanagerContext, + definitions: &ValidatorDefinitions, +) -> crate::Result<()> { + let saved = definitions.save(&context.validators_dir); + match &saved { + Ok(()) => context + .definitions_unflushed + .store(false, std::sync::atomic::Ordering::Release), + Err(Error::Unflushed { .. }) => context + .definitions_unflushed + .store(true, std::sync::atomic::Ordering::Release), + Err(_) => {} + } + saved +} + +/// The keystore and password paths `persist_import` writes `pubkey` to. +fn written_paths(context: &KeymanagerContext, pubkey: &BlsPubkey) -> crate::Result<[PathBuf; 2]> { + let base = encode_hex(&pubkey.0); + Ok([ + absolute(context.validators_dir.join(format!("{base}.json")))?, + absolute(context.secrets_dir.join(&base))?, + ]) +} + +/// Remove `path`; one that is already gone is not an error. +fn remove_if_present(path: &Path) -> crate::Result<()> { + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(Error::Io { + path: path.display().to_string(), + source, + }), + } +} + /// Decrypt one keystore and persist it to disk: everything an import can do -/// before it needs the store's write lock. +/// before it needs the store's write lock and its slashing-protection record. fn prepare_import( context: &KeymanagerContext, definitions: &mut ValidatorDefinitions, json: &str, password: Option<&str>, -) -> std::result::Result, KeyStatus> { +) -> std::result::Result<(BlsPubkey, Zeroizing<[u8; 32]>, Overwritten), KeyStatus> { let Some(password) = password else { return Err(KeyStatus::error("no password supplied for this keystore")); }; @@ -248,9 +562,39 @@ fn prepare_import( // crash right after "imported" is reported silently loses it, with // nothing about a live, signing validator hinting that it is about to // vanish on the next restart. - persist_import(context, definitions, &pubkey, json, password).map_err(KeyStatus::error)?; + let overwritten = + persist_import(context, definitions, &pubkey, json, password).map_err(KeyStatus::error)?; + + Ok((pubkey, secret, overwritten)) +} + +/// What `persist_import` found at the two paths it writes, so an undone +/// import can put it back: the bytes of a regular file that was there, or +/// `None` for nothing. Restored as a pair, so a keystore never ends up next +/// to a password it was not encrypted under. +struct Overwritten { + keystore: Option>, + password: Option>>, +} - Ok(secret) +/// The bytes of the regular file at `path`, or `None` when there is none. A +/// symlink or a directory reads as `None` too: the write that follows refuses +/// those, so nothing of theirs is overwritten. +fn regular_file_bytes(path: &Path) -> crate::Result>> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_file() => { + std::fs::read(path).map(Some).map_err(|source| Error::Io { + path: path.display().to_string(), + source, + }) + } + Ok(_) => Ok(None), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(Error::Io { + path: path.display().to_string(), + source, + }), + } } /// Add an already-decrypted, already-persisted key to the live store. The @@ -277,7 +621,7 @@ fn persist_import( pubkey: &BlsPubkey, keystore_json: &str, password: &str, -) -> crate::Result<()> { +) -> crate::Result { // Named after the pubkey, the convention staking tools use, so an // operator can tell which file backs which validator without opening it. // That predictability is exactly why the writes below refuse to follow a @@ -298,22 +642,46 @@ fn persist_import( // file, so re-importing the same key (rotating its password, or simply // retrying) behaves exactly as a plain `std::fs::write` would; it only // refuses to follow a symlink planted at the path ahead of time. - secure_fs::write_private_no_symlink(&keystore_path, keystore_json).map_err(|source| { - Error::Io { + // Taken before the writes, to be put back if this import is undone. + let overwritten = Overwritten { + keystore: regular_file_bytes(&keystore_path)?, + password: regular_file_bytes(&password_path)?.map(Zeroizing::new), + }; + // Both files, or neither: a failure after the first write puts both back + // as they were, so a re-import that fails on its second file does not + // leave the previous keystore destroyed under the previous entry. + let restore = |overwritten: &Overwritten| -> crate::Result<()> { + put_back(&keystore_path, overwritten.keystore.as_deref())?; + put_back( + &password_path, + overwritten.password.as_deref().map(|bytes| &bytes[..]), + ) + }; + let written = secure_fs::write_private_no_symlink(&keystore_path, keystore_json) + .map_err(|source| Error::Io { path: keystore_path.display().to_string(), source, - } - })?; - secure_fs::write_private_no_symlink(&password_path, password).map_err(|source| Error::Io { - path: password_path.display().to_string(), - source, - })?; + }) + .and_then(|()| { + secure_fs::write_private_no_symlink(&password_path, password).map_err(|source| { + Error::Io { + path: password_path.display().to_string(), + source, + } + }) + }); + if let Err(err) = written { + restore(&overwritten).inspect_err( + |undo| tracing::error!(%undo, "Could not put a half-written keystore back"), + )?; + return Err(err); + } let definition = ValidatorDefinition { enabled: true, voting_public_key: base.clone(), - voting_keystore_path: keystore_path, - voting_keystore_password_path: password_path, + voting_keystore_path: keystore_path.clone(), + voting_keystore_password_path: password_path.clone(), }; // How to put `definitions` back if the save below fails. // @@ -321,9 +689,9 @@ fn persist_import( // and re-saves it. Leaving a failed key's mutation in place would let the // *next* key's successful save write it to disk, so a key this request // reports as `error` would be enabled on the next restart and start - // signing. With no slashing-protection record, a validator the operator - // believes was never imported is exactly the kind that ends up running in - // two places. + // signing. A validator the operator believes was never imported is exactly + // the kind that ends up running in two places, which no local record can + // see. enum Undo { Appended, Replaced { @@ -351,19 +719,36 @@ fn persist_import( } }; - if let Err(err) = definitions.save(&context.validators_dir) { - // `Appended` pops rather than removing by index because this function - // pushes and saves with nothing in between, so the entry it added is - // still the last one. - match undo { - Undo::Appended => { - definitions.0.pop(); + match save_definitions(context, definitions) { + Ok(()) => Ok(overwritten), + // The entry is on disk and every reader sees it; only its survival + // of a power loss is in doubt, and an entry lost that way only + // removes a key. Counted as persisted, and kept in memory, so what + // this request holds matches the file. + Err(Error::Unflushed { path, source }) => { + warn!( + %path, + %source, + "Validator definitions written, but the directory could not be flushed" + ); + Ok(overwritten) + } + Err(err) => { + // `Appended` pops rather than removing by index because this + // function pushes and saves with nothing in between, so the + // entry it added is still the last one. The files go back too. + match undo { + Undo::Appended => { + definitions.0.pop(); + } + Undo::Replaced { index, previous } => definitions.0[index] = previous, } - Undo::Replaced { index, previous } => definitions.0[index] = previous, + restore(&overwritten).inspect_err(|undo| { + tracing::error!(%undo, "Could not put a keystore whose entry failed to save back") + })?; + Err(err) } - return Err(err); } - Ok(()) } fn absolute(path: PathBuf) -> crate::Result { @@ -385,29 +770,93 @@ pub async fn delete( let _definitions_guard = context.definitions_lock.lock().await; let mut store = context.store.write().await; - let data = request + let results: Vec<(Option, KeyStatus)> = request .pubkeys .iter() .map(|text| match parse_pubkey(text) { - Ok(pubkey) => delete_one(&context, &mut store, &pubkey), - Err(err) => KeyStatus::error(err), + Ok(pubkey) => (Some(pubkey), delete_one(&context, &mut store, &pubkey)), + Err(err) => (None, KeyStatus::error(err)), }) .collect(); drop(store); - // This interchange cannot be used to migrate the key(s) just deleted to - // another client: see the module doc for why `genesis_validators_root` - // is deliberately wrong rather than merely absent. Logged on every call, - // not just a failure, since a successful response is exactly the one a - // caller might mistake for "safe to import elsewhere". - warn!( - "DELETE /eth/v1/keystores returns an empty EIP-3076 interchange with no real signing \ - history; it cannot be used to migrate these keys to another client without slashing risk" - ); + // Exported only now, with the keys out of the store: a signature with one + // of them either finished before the write lock above was granted, and is + // in the record, or can no longer happen. That holds because the duty + // services record before they take the store's read lock to sign (see + // `attestation.rs` and `proposal.rs`), so no signature exists without its + // record already on disk. Only for the keys that are `deleted` or may + // become `not_active`, as the specification requires: a key whose delete + // failed is still on disk and comes back on restart, so its history is + // not handed over, and neither is one whose removal reached the disk but + // not a flush (`Error::Unflushed`, an `error` too), since it may come + // back. Teku and Prysm filter the same way. A key listed twice is + // `deleted` once and `not_found` once; `export` lists it once. + let requested: Vec> = results + .iter() + .map(|(pubkey, status)| match status.status.as_str() { + "deleted" | "not_found" => *pubkey, + _ => None, + }) + .collect(); + let exported = crate::slashing_protection::blocking(&context.protection, move |protection| { + let pubkeys: Vec = requested.iter().flatten().copied().collect(); + let interchange = protection.export(&pubkeys)?; + let recorded: Vec = requested + .iter() + .map(|pubkey| pubkey.is_some_and(|pubkey| protection.contains(&pubkey))) + .collect(); + Ok((interchange, recorded)) + }) + .await; + let (slashing_protection, recorded) = match exported { + Ok(exported) => exported, + Err(err) => { + // With the record unreadable, nothing here can tell a key with no + // history from one whose history could not be read, so every key + // that parsed becomes an error, as Prysm answers: a `not_found` + // here would tell a migrator there is nothing to carry. A + // malformed pubkey keeps its own error. The deletions themselves + // already happened and are not undone: the keys' records are + // kept, so a retry answers `not_active` with their history. The + // field is required, so it carries an empty string rather than + // an interchange that would read as "nothing signed". + warn!(%err, "Deleted keys whose slashing-protection history could not be exported"); + let message = format!("slashing_protection could not be exported: {err}"); + let data = results + .into_iter() + .map(|(pubkey, status)| match pubkey { + Some(_) => KeyStatus::error(&message), + None => status, + }) + .collect(); + return Json(DeleteResponse { + data, + slashing_protection: String::new(), + }); + } + }; + + // `not_active`: not held here, but with a record, which the response + // carries. The specification's word for "gone, history attached". + let data = results + .into_iter() + .zip(recorded) + .map(|((_, status), recorded)| { + if status.status == "not_found" && recorded { + KeyStatus { + status: "not_active".to_string(), + message: None, + } + } else { + status + } + }) + .collect(); Json(DeleteResponse { data, - slashing_protection: empty_interchange(), + slashing_protection, }) } @@ -435,6 +884,25 @@ fn delete_one( context.settings.forget(pubkey); if !on_disk && !in_memory { + // A removal whose directory flush failed is on disk but may not + // survive a power loss, and a retry lands here. Answering `not_found` + // then would tell the operator the key is gone for good; flush first, + // by saving the file again, and keep the error until that works. + if context + .definitions_unflushed + .load(std::sync::atomic::Ordering::Acquire) + { + let definitions = match ValidatorDefinitions::open(&context.validators_dir) { + Ok(definitions) => definitions, + Err(err) => return KeyStatus::error(err), + }; + if let Err(err) = save_definitions(context, &definitions) { + return KeyStatus::error(format!( + "an earlier removal is on disk but the directory could not be flushed: \ + {err}; the key may come back after a power loss" + )); + } + } return KeyStatus { status: "not_found".to_string(), message: None, @@ -453,8 +921,19 @@ fn delete_one( // a `not_found` that reads as "already gone": acting on that, by importing // the key elsewhere, is what turns a failed delete into two hosts signing // for one validator. - if let Err(err) = persist_delete(context, pubkey) { - return KeyStatus::error(err); + match persist_delete(context, pubkey) { + Ok(()) => {} + // The removal is on disk but may not survive a power loss, and the + // keystore stays, so the key could come back enabled, with its + // record, while the operator runs it elsewhere. An error, so no + // caller takes the key for gone; this process does not sign with it. + Err(Error::Unflushed { path, source }) => { + return KeyStatus::error(format!( + "removed from {path}, but the directory could not be flushed: {source}; the key \ + may come back after a power loss" + )); + } + Err(err) => return KeyStatus::error(err), } info!(pubkey = %encode_hex(&pubkey.0), "Deleted validator key"); @@ -497,24 +976,7 @@ fn persist_delete(context: &KeymanagerContext, pubkey: &BlsPubkey) -> crate::Res .map(|parsed| parsed != *pubkey) .unwrap_or(true) }); - definitions.save(&context.validators_dir) -} - -/// A well-formed EIP-3076 interchange holding no history. -/// -/// `genesis_validators_root` is deliberately all-zero rather than the -/// chain's real value; see the module doc for why. Do not "fix" this to -/// carry the real root: that would make an empty history look legitimate -/// instead of getting it rejected. -fn empty_interchange() -> String { - serde_json::json!({ - "metadata": { - "interchange_format_version": INTERCHANGE_VERSION, - "genesis_validators_root": "0x0000000000000000000000000000000000000000000000000000000000000000" - }, - "data": [] - }) - .to_string() + save_definitions(context, &definitions) } #[cfg(test)] @@ -564,6 +1026,8 @@ mod tests { validators_dir: dir.to_path_buf(), secrets_dir: dir.to_path_buf(), definitions_lock: Arc::new(tokio::sync::Mutex::new(())), + definitions_unflushed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + protection: crate::slashing_protection::SlashingProtection::in_memory(), } } @@ -900,6 +1364,761 @@ mod tests { ); } + /// An imported key is registered in the slashing-protection record before + /// it is loaded, so the next start finds a record for it instead of + /// refusing it. + #[tokio::test] + async fn an_imported_key_is_registered_for_slashing_protection() { + let dir = tempfile::tempdir().expect("temp dir"); + let root = ethlambda_types::beacon::primitives::Root::ZERO; + let protection = + crate::slashing_protection::SlashingProtection::open(dir.path(), root, &[], false) + .expect("opens with no keys"); + let mut context = + context_with_dirs(Arc::new(RwLock::new(ValidatorStore::new())), dir.path()); + context.protection = Arc::new(std::sync::Mutex::new(protection)); + let protection = context.protection.clone(); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "imported", "got {body}"); + assert_eq!(protection.lock().expect("not poisoned").len(), 1); + + // The next start: same directory, the imported key loaded, no + // --init-slashing-protection. It must open, because the key is + // registered. + drop(protection); + let keys = ValidatorStore::load(dir.path()).expect("loads").pubkeys(); + crate::slashing_protection::SlashingProtection::open(dir.path(), root, &keys, false) + .expect("the imported key has a record"); + } + + /// A key whose registration fails is taken off disk again, so the failed + /// write costs this request only and the next start loads nothing it + /// would refuse. + #[tokio::test] + async fn a_failed_registration_rolls_the_import_back() { + let dir = tempfile::tempdir().expect("temp dir"); + let root = ethlambda_types::beacon::primitives::Root::ZERO; + let protection = + crate::slashing_protection::SlashingProtection::open(dir.path(), root, &[], false) + .expect("opens with no keys"); + // A directory in the record's place makes the registration's rename + // fail, whoever runs the test. + let record = protection.path().expect("persisted").to_path_buf(); + std::fs::remove_file(&record).expect("removes the record"); + std::fs::create_dir(&record).expect("a directory in its place"); + let mut context = + context_with_dirs(Arc::new(RwLock::new(ValidatorStore::new())), dir.path()); + context.protection = Arc::new(std::sync::Mutex::new(protection)); + let store = context.store.clone(); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert!( + body["data"][0]["message"] + .as_str() + .unwrap_or_default() + .starts_with("not imported"), + "got {body}" + ); + assert!(store.read().await.pubkeys().is_empty(), "not loaded"); + assert!( + ValidatorStore::load(dir.path()) + .expect("loads") + .pubkeys() + .is_empty(), + "the next start finds no key to refuse" + ); + let leftovers: Vec = std::fs::read_dir(dir.path()) + .expect("lists") + .filter_map(|entry| entry.ok()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| { + !name.starts_with("slashing_protection") + && name != crate::keys::definitions::DEFINITIONS_FILE + }) + .collect(); + assert!( + leftovers.is_empty(), + "keystore and password removed: {leftovers:?}" + ); + } + + /// A failed registration puts back the definitions entry the request + /// replaced: a disabled entry, which was never loaded and has no record, + /// stays disabled rather than enabled on disk, which would refuse the + /// whole client at the next start. + #[tokio::test] + async fn a_failed_registration_restores_a_replaced_definitions_entry() { + let dir = tempfile::tempdir().expect("temp dir"); + let root = ethlambda_types::beacon::primitives::Root::ZERO; + let protection = + crate::slashing_protection::SlashingProtection::open(dir.path(), root, &[], false) + .expect("opens with no keys"); + let record = protection.path().expect("persisted").to_path_buf(); + std::fs::remove_file(&record).expect("removes the record"); + std::fs::create_dir(&record).expect("a directory in its place"); + let keystore: serde_json::Value = + serde_json::from_str(&keystore_json()).expect("the vector is json"); + let previous = ValidatorDefinition { + enabled: false, + voting_public_key: format!("0x{}", keystore["pubkey"].as_str().expect("pubkey")), + voting_keystore_path: dir.path().join("old.json"), + voting_keystore_password_path: dir.path().join("old"), + }; + ValidatorDefinitions(vec![previous.clone()]) + .save(dir.path()) + .expect("saves"); + let mut context = + context_with_dirs(Arc::new(RwLock::new(ValidatorStore::new())), dir.path()); + context.protection = Arc::new(std::sync::Mutex::new(protection)); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert!( + body["data"][0]["message"] + .as_str() + .unwrap_or_default() + .contains("back as it was"), + "got {body}" + ); + let definitions = ValidatorDefinitions::open(dir.path()).expect("opens"); + assert_eq!(definitions.0, vec![previous], "the disabled entry is back"); + let json_files: Vec = std::fs::read_dir(dir.path()) + .expect("lists") + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.ends_with(".json")) + .collect(); + assert!( + json_files.is_empty(), + "the keystore this request wrote goes back to what was there, nothing: {json_files:?}" + ); + } + + /// A failed registration puts the keystore and password files a re-import + /// overwrote back as they were, as a pair, so the restored entry never + /// points at a keystore encrypted under a password that is gone. + #[tokio::test] + async fn a_failed_registration_restores_the_overwritten_files() { + let dir = tempfile::tempdir().expect("temp dir"); + let root = ethlambda_types::beacon::primitives::Root::ZERO; + let protection = + crate::slashing_protection::SlashingProtection::open(dir.path(), root, &[], false) + .expect("opens with no keys"); + let record = protection.path().expect("persisted").to_path_buf(); + std::fs::remove_file(&record).expect("removes the record"); + std::fs::create_dir(&record).expect("a directory in its place"); + let keystore: serde_json::Value = + serde_json::from_str(&keystore_json()).expect("the vector is json"); + let pubkey = parse_pubkey(keystore["pubkey"].as_str().expect("pubkey")).expect("parses"); + let base = encode_hex(&pubkey.0); + let keystore_path = dir.path().join(format!("{base}.json")); + let password_path = dir.path().join(&base); + std::fs::write(&keystore_path, b"old keystore").expect("writes"); + std::fs::write(&password_path, b"old password").expect("writes"); + let previous = ValidatorDefinition { + enabled: true, + voting_public_key: base.clone(), + voting_keystore_path: keystore_path.clone(), + voting_keystore_password_path: password_path.clone(), + }; + ValidatorDefinitions(vec![previous.clone()]) + .save(dir.path()) + .expect("saves"); + let mut context = + context_with_dirs(Arc::new(RwLock::new(ValidatorStore::new())), dir.path()); + context.protection = Arc::new(std::sync::Mutex::new(protection)); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert_eq!( + std::fs::read(&keystore_path).expect("still there"), + b"old keystore" + ); + assert_eq!( + std::fs::read(&password_path).expect("still there"), + b"old password" + ); + let definitions = ValidatorDefinitions::open(dir.path()).expect("opens"); + assert_eq!(definitions.0, vec![previous]); + } + + /// A re-import whose password write fails puts the keystore it had already + /// replaced back, so the previous entry never points at a keystore + /// encrypted under a password that was never written. + #[tokio::test] + async fn a_failed_password_write_puts_the_replaced_keystore_back() { + let dir = tempfile::tempdir().expect("temp dir"); + let keystore: serde_json::Value = + serde_json::from_str(&keystore_json()).expect("the vector is json"); + let pubkey = parse_pubkey(keystore["pubkey"].as_str().expect("pubkey")).expect("parses"); + let base = encode_hex(&pubkey.0); + let keystore_path = dir.path().join(format!("{base}.json")); + let password_path = dir.path().join(&base); + std::fs::write(&keystore_path, b"old keystore").expect("writes"); + // A directory at the password's path makes its write fail, after the + // keystore's succeeded. + std::fs::create_dir(&password_path).expect("a directory in its place"); + let previous = ValidatorDefinition { + enabled: true, + voting_public_key: base.clone(), + voting_keystore_path: keystore_path.clone(), + voting_keystore_password_path: dir.path().join("old"), + }; + ValidatorDefinitions(vec![previous.clone()]) + .save(dir.path()) + .expect("saves"); + let context = context_with_dirs(Arc::new(RwLock::new(ValidatorStore::new())), dir.path()); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert_eq!( + std::fs::read(&keystore_path).expect("still there"), + b"old keystore" + ); + let definitions = ValidatorDefinitions::open(dir.path()).expect("opens"); + assert_eq!(definitions.0, vec![previous]); + } + + /// A request that carries history is refused whole, importing nothing. + #[tokio::test] + async fn an_import_carrying_history_is_refused_whole() { + let dir = tempfile::tempdir().expect("temp dir"); + let context = context_with_dirs(Arc::new(RwLock::new(ValidatorStore::new())), dir.path()); + let store = context.store.clone(); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + "slashing_protection": "{}", + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert!(store.read().await.pubkeys().is_empty()); + assert!( + ValidatorDefinitions::open(dir.path()) + .expect("opens") + .0 + .is_empty() + ); + } + + /// The same keystore twice in one request, with the registration failing: + /// the second copy snapshotted what the first one wrote, and the undo must + /// still end with what was there before the request. + #[tokio::test] + async fn a_failed_registration_restores_the_state_before_a_duplicated_import() { + let dir = tempfile::tempdir().expect("temp dir"); + let root = ethlambda_types::beacon::primitives::Root::ZERO; + let protection = + crate::slashing_protection::SlashingProtection::open(dir.path(), root, &[], false) + .expect("opens with no keys"); + let record = protection.path().expect("persisted").to_path_buf(); + std::fs::remove_file(&record).expect("removes the record"); + std::fs::create_dir(&record).expect("a directory in its place"); + let mut context = + context_with_dirs(Arc::new(RwLock::new(ValidatorStore::new())), dir.path()); + context.protection = Arc::new(std::sync::Mutex::new(protection)); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json(), keystore_json()], + "passwords": [PASSWORD, PASSWORD], + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert_eq!(body["data"][1]["status"], "error", "got {body}"); + assert!( + ValidatorDefinitions::open(dir.path()) + .expect("opens") + .0 + .is_empty() + ); + let leftovers: Vec = std::fs::read_dir(dir.path()) + .expect("lists") + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| { + !name.starts_with("slashing_protection") + && name != crate::keys::definitions::DEFINITIONS_FILE + }) + .collect(); + assert!( + leftovers.is_empty(), + "nothing was there before: {leftovers:?}" + ); + } + + /// A key that fails to persist must leave no slashing-protection record + /// behind: a record would later let the same keystore, copied in by hand, + /// past the startup check as if it had been registered on purpose. + #[cfg(unix)] + #[tokio::test] + async fn a_failed_import_leaves_no_slashing_protection_record() { + let dir = tempfile::tempdir().expect("temp dir"); + // A symlink at the keystore's path makes `persist_import` refuse. + let target = dir.path().join("elsewhere"); + std::fs::write(&target, "untouched").expect("writes target"); + let pubkey = "0x9612d7a727c9d0a22e185a1c768478dfe919cada9266988cb32359c11f2b7b27f4ae4040902382ae2910c15e2b420d07"; + std::os::unix::fs::symlink(&target, dir.path().join(format!("{pubkey}.json"))) + .expect("symlinks"); + + let root = ethlambda_types::beacon::primitives::Root::ZERO; + let protection = + crate::slashing_protection::SlashingProtection::open(dir.path(), root, &[], false) + .expect("opens with no keys"); + let mut context = + context_with_dirs(Arc::new(RwLock::new(ValidatorStore::new())), dir.path()); + context.protection = Arc::new(std::sync::Mutex::new(protection)); + let protection = context.protection.clone(); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ "keystores": [keystore_json()], "passwords": [PASSWORD] }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert!( + protection.lock().expect("not poisoned").is_empty(), + "the failed key must not have been registered" + ); + } + + const KEYSTORE_PUBKEY: &str = "0x9612d7a727c9d0a22e185a1c768478dfe919cada9266988cb32359c11f2b7b27f4ae4040902382ae2910c15e2b420d07"; + + /// A context over `dir` whose slashing protection is a real record there, + /// for the chain `0x00..00`, and the handle to inspect it with. + fn context_with_record( + dir: &std::path::Path, + ) -> ( + KeymanagerContext, + crate::slashing_protection::SharedProtection, + ) { + let protection = crate::slashing_protection::SlashingProtection::open( + dir, + ethlambda_types::beacon::primitives::Root::ZERO, + &[], + false, + ) + .expect("opens with no keys"); + let mut context = context_with_dirs(Arc::new(RwLock::new(ValidatorStore::new())), dir); + context.protection = Arc::new(std::sync::Mutex::new(protection)); + let protection = context.protection.clone(); + (context, protection) + } + + /// A minimal interchange for `KEYSTORE_PUBKEY` on the chain `root`. + fn history(root: &str, slot: u64, source: u64, target: u64) -> String { + history_for(KEYSTORE_PUBKEY, root, slot, source, target) + } + + fn history_for(pubkey: &str, root: &str, slot: u64, source: u64, target: u64) -> String { + serde_json::json!({ + "metadata": { "interchange_format_version": "5", "genesis_validators_root": root }, + "data": [{ + "pubkey": pubkey, + "signed_blocks": [{ "slot": slot.to_string() }], + "signed_attestations": [{ + "source_epoch": source.to_string(), + "target_epoch": target.to_string() + }] + }] + }) + .to_string() + } + + const ZERO_ROOT: &str = "0x0000000000000000000000000000000000000000000000000000000000000000"; + + /// History that arrives with a key is in force before the key can sign. + #[tokio::test] + async fn an_import_brings_its_slashing_protection_history() { + let dir = tempfile::tempdir().expect("temp dir"); + let (context, protection) = context_with_record(dir.path()); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + "slashing_protection": history(ZERO_ROOT, 100, 5, 10), + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "imported", "got {body}"); + + let pubkey = parse_pubkey(KEYSTORE_PUBKEY).expect("pubkey"); + let protection = protection.lock().expect("not poisoned"); + assert!( + protection.check_block(&pubkey, 100).is_err(), + "slot 100 was signed elsewhere" + ); + assert!(protection.check_block(&pubkey, 101).is_ok()); + } + + /// History for another chain is refused, and with it every key in the + /// request: none may start signing without the history it came with. + #[tokio::test] + async fn an_import_whose_history_is_for_another_chain_imports_nothing() { + let dir = tempfile::tempdir().expect("temp dir"); + let (context, _) = context_with_record(dir.path()); + let store = context.store.clone(); + let other_chain = "0x0000000000000000000000000000000000000000000000000000000000000001"; + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + "slashing_protection": history(other_chain, 100, 5, 10), + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert!(store.read().await.is_empty(), "the key must not be loaded"); + } + + /// A delete hands over the record the key was protected by, so it can be + /// imported elsewhere, and keeps it here. + #[tokio::test] + async fn a_delete_returns_the_keys_history() { + let dir = tempfile::tempdir().expect("temp dir"); + let (context, protection) = context_with_record(dir.path()); + let app = router(context, TOKEN.to_string()); + + let (_, body) = send( + app.clone(), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ "keystores": [keystore_json()], "passwords": [PASSWORD] }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "imported", "got {body}"); + // Imported with no history, so the key starts at the current slot; + // a proposal must come after it. + const SLOT: u64 = 1 << 40; + let pubkey = parse_pubkey(KEYSTORE_PUBKEY).expect("pubkey"); + protection + .lock() + .expect("not poisoned") + .check_and_record_block(&pubkey, SLOT) + .expect("written") + .expect("allowed"); + + let (_, body) = send( + app.clone(), + authed( + "DELETE", + "/eth/v1/keystores", + serde_json::json!({ "pubkeys": [KEYSTORE_PUBKEY] }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "deleted", "got {body}"); + let interchange: serde_json::Value = + serde_json::from_str(body["slashing_protection"].as_str().expect("a string")) + .expect("json"); + assert_eq!( + interchange["metadata"]["genesis_validators_root"], + ZERO_ROOT + ); + assert_eq!(interchange["data"][0]["pubkey"], KEYSTORE_PUBKEY); + assert_eq!( + interchange["data"][0]["signed_blocks"][0]["slot"], + SLOT.to_string() + ); + + // Gone from this client, but its record is still here and still + // returned: the specification's `not_active`. + let (_, body) = send( + app, + authed( + "DELETE", + "/eth/v1/keystores", + serde_json::json!({ "pubkeys": [KEYSTORE_PUBKEY] }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "not_active", "got {body}"); + assert!( + protection + .lock() + .expect("not poisoned") + .check_block(&pubkey, SLOT) + .is_err() + ); + } + + /// The history may name a validator that is not in the request: it gets a + /// record too, so a later delete of it answers `not_active` with that + /// history rather than `not_found`. + #[tokio::test] + async fn history_for_a_key_outside_the_request_gets_a_record() { + let dir = tempfile::tempdir().expect("temp dir"); + let (context, protection) = context_with_record(dir.path()); + let app = router(context, TOKEN.to_string()); + let other = format!("0x{}", "ab".repeat(48)); + + let (_, body) = send( + app.clone(), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + "slashing_protection": history_for(&other, ZERO_ROOT, 100, 5, 10), + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "imported", "got {body}"); + + let other_pubkey = parse_pubkey(&other).expect("pubkey"); + { + let protection = protection.lock().expect("not poisoned"); + assert!(protection.contains(&other_pubkey)); + assert!(protection.check_block(&other_pubkey, 100).is_err()); + assert!(protection.contains(&parse_pubkey(KEYSTORE_PUBKEY).expect("pubkey"))); + } + + let (_, body) = send( + app, + authed( + "DELETE", + "/eth/v1/keystores", + serde_json::json!({ "pubkeys": [other] }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "not_active", "got {body}"); + let interchange: serde_json::Value = + serde_json::from_str(body["slashing_protection"].as_str().expect("a string")) + .expect("json"); + assert_eq!(interchange["data"][0]["pubkey"], other); + assert_eq!(interchange["data"][0]["signed_blocks"][0]["slot"], "100"); + } + + /// A key listed twice in one delete is `deleted` once and `not_active` + /// once, and the returned interchange names it once: one that named a + /// validator twice would be refused on import. + #[tokio::test] + async fn a_delete_listing_a_key_twice_exports_it_once() { + let dir = tempfile::tempdir().expect("temp dir"); + let (context, _) = context_with_record(dir.path()); + let app = router(context, TOKEN.to_string()); + + let (_, body) = send( + app.clone(), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ "keystores": [keystore_json()], "passwords": [PASSWORD] }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "imported", "got {body}"); + + let (_, body) = send( + app, + authed( + "DELETE", + "/eth/v1/keystores", + serde_json::json!({ "pubkeys": [KEYSTORE_PUBKEY, KEYSTORE_PUBKEY] }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "deleted", "got {body}"); + assert_eq!(body["data"][1]["status"], "not_active", "got {body}"); + let interchange: serde_json::Value = + serde_json::from_str(body["slashing_protection"].as_str().expect("a string")) + .expect("json"); + assert_eq!(interchange["data"].as_array().expect("array").len(), 1); + assert_eq!(interchange["data"][0]["pubkey"], KEYSTORE_PUBKEY); + } + + /// The history is written before its key is saved, so it is held even + /// when the key then fails to save. The key here was deleted earlier and + /// kept an older record: had the key been saved first, a crash before the + /// history landed would have loaded it on that older record at the next + /// start, which is the case the ordering exists for. + #[cfg(unix)] + #[tokio::test] + async fn history_is_written_before_its_key_is_saved() { + let dir = tempfile::tempdir().expect("temp dir"); + // A symlink at the keystore's path makes `persist_import` refuse. + let target = dir.path().join("elsewhere"); + std::fs::write(&target, "untouched").expect("writes target"); + std::os::unix::fs::symlink(&target, dir.path().join(format!("{KEYSTORE_PUBKEY}.json"))) + .expect("symlinks"); + let (context, protection) = context_with_record(dir.path()); + let pubkey = parse_pubkey(KEYSTORE_PUBKEY).expect("pubkey"); + { + let mut protection = protection.lock().expect("not poisoned"); + protection.register(&[pubkey]).expect("registers"); + protection + .check_and_record_block(&pubkey, 10) + .expect("written") + .expect("allowed"); + } + let store = context.store.clone(); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + "slashing_protection": history(ZERO_ROOT, 100, 5, 10), + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert!(store.read().await.is_empty(), "the key did not load"); + assert!( + !dir.path().join("validator_definitions.yml").exists(), + "nothing was saved" + ); + let protection = protection.lock().expect("not poisoned"); + assert!( + protection.check_block(&pubkey, 100).is_err(), + "the history raised the key's record" + ); + assert!(protection.check_block(&pubkey, 101).is_ok()); + } + + /// The history is written before any key is touched: when that write + /// fails, no key is saved or loaded, so nothing of the request is on disk + /// to load and sign after a restart as if the import had half succeeded. + #[tokio::test] + async fn a_failed_history_write_saves_no_key() { + let dir = tempfile::tempdir().expect("temp dir"); + let (context, protection) = context_with_record(dir.path()); + // A directory in the record's place makes the write's rename fail. + let record = protection + .lock() + .expect("not poisoned") + .path() + .expect("persisted") + .to_path_buf(); + std::fs::remove_file(&record).expect("removes the record"); + std::fs::create_dir(&record).expect("a directory in its place"); + + let (_, body) = send( + router(context, TOKEN.to_string()), + authed( + "POST", + "/eth/v1/keystores", + serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + "slashing_protection": history(ZERO_ROOT, 100, 5, 10), + }), + ), + ) + .await; + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert!( + body["data"][0]["message"] + .as_str() + .unwrap_or_default() + .starts_with("slashing_protection was not imported"), + "got {body}" + ); + assert!(protection.lock().expect("not poisoned").is_empty()); + assert!( + !dir.path().join(format!("{KEYSTORE_PUBKEY}.json")).exists(), + "no keystore was saved" + ); + } + #[tokio::test] async fn deleting_an_unknown_key_reports_not_found() { let (_, body) = send( @@ -1242,20 +2461,6 @@ mod tests { .to_string() } - /// Finding 6: the interchange's `genesis_validators_root` must stay - /// all-zero. See the module doc: a real root would make an empty history - /// look legitimate, which is worse than the deliberately unusable file - /// this client returns instead. - #[test] - fn the_empty_interchange_root_stays_all_zero() { - let interchange: serde_json::Value = - serde_json::from_str(&empty_interchange()).expect("valid json"); - assert_eq!( - interchange["metadata"]["genesis_validators_root"], - "0x0000000000000000000000000000000000000000000000000000000000000000", - ); - } - /// Finding 3: `require_bearer` must reject a wrong token of a different /// length too, not just one that fails a byte comparison partway /// through. Not a timing assertion (this crate has no timing harness), diff --git a/crates/validator/src/http_api/mod.rs b/crates/validator/src/http_api/mod.rs index 26e250b6e..189e5ac7a 100644 --- a/crates/validator/src/http_api/mod.rs +++ b/crates/validator/src/http_api/mod.rs @@ -58,6 +58,14 @@ pub struct KeymanagerContext { /// it takes this one first and `store` second, consistently, so the two /// never deadlock against each other. pub definitions_lock: Arc>, + /// Whether the last save of the definitions file replaced it without + /// flushing the directory, so its last change may not survive a power + /// loss. Set and cleared by `keystores::save_definitions`; read where a + /// retry would otherwise report that change as final. + pub definitions_unflushed: Arc, + /// The record every key must be registered in before it is loaded, so the + /// duty path never meets a key it has no record for. + pub protection: crate::slashing_protection::SharedProtection, } /// The file the generated token is written to, inside the validators directory. @@ -155,8 +163,16 @@ pub fn router(context: KeymanagerContext, token: String) -> Router { .with_state(context) .layer(middleware::from_fn(require_bearer)) .layer(Extension(ApiToken(token))) + // `POST /eth/v1/keystores` carries a whole EIP-3076 interchange, which + // in the complete format for a batch of validators runs past axum's + // 2 MB default. Nimbus allows 16 MiB and Lodestar 20 MB; Lighthouse + // and Prysm set no limit. + .layer(axum::extract::DefaultBodyLimit::max(KEYMANAGER_BODY_LIMIT)) } +/// The largest request body the keymanager API reads. +const KEYMANAGER_BODY_LIMIT: usize = 64 * 1024 * 1024; + #[derive(Clone)] struct ApiToken(String); diff --git a/crates/validator/src/http_api/proposer.rs b/crates/validator/src/http_api/proposer.rs index beec84d3c..91c5ab5e4 100644 --- a/crates/validator/src/http_api/proposer.rs +++ b/crates/validator/src/http_api/proposer.rs @@ -367,6 +367,8 @@ mod tests { validators_dir: dir.path().to_path_buf(), secrets_dir: dir.path().to_path_buf(), definitions_lock: Arc::new(tokio::sync::Mutex::new(())), + definitions_unflushed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + protection: crate::slashing_protection::SlashingProtection::in_memory(), }; Harness { app: router(context, TOKEN.to_string()), diff --git a/crates/validator/src/keys/definitions.rs b/crates/validator/src/keys/definitions.rs index ab48a8db0..3b091585a 100644 --- a/crates/validator/src/keys/definitions.rs +++ b/crates/validator/src/keys/definitions.rs @@ -51,12 +51,11 @@ impl ValidatorDefinitions { /// Write the definitions back, replacing the file. /// - /// Writes to a sibling temporary file and renames it over the target - /// rather than truncating in place. The keymanager API calls this while - /// validators are signing, and a crash partway through an in-place write - /// would leave a truncated file that stops the client booting, losing every - /// validator rather than the one being changed. `rename` within one - /// directory is atomic, so a reader sees the old file or the new one. + /// Through [`secure_fs::write_private_durably`] rather than truncating in + /// place. The keymanager API calls this while validators are signing, and + /// a crash partway through an in-place write would leave a truncated file + /// that stops the client booting, losing every validator rather than the + /// one being changed. pub fn save(&self, validators_dir: &Path) -> Result<()> { let path = validators_dir.join(DEFINITIONS_FILE); let contents = serde_yaml_ng::to_string(self).map_err(|err| Error::Keystore { @@ -64,17 +63,24 @@ impl ValidatorDefinitions { reason: err.to_string(), })?; - // Mode set on the temporary file, not the final path: `rename` - // preserves it, and setting it here means the file is never - // world-readable even for the instant between creation and rename. - let temporary = validators_dir.join(format!("{DEFINITIONS_FILE}.tmp")); - secure_fs::write_private(&temporary, contents).map_err(|source| Error::Io { - path: temporary.display().to_string(), - source, - })?; - std::fs::rename(&temporary, &path).map_err(|source| Error::Io { - path: path.display().to_string(), - source, + // An after-rename failure is `Error::Unflushed`: the file is replaced + // and every reader sees the new one, only its survival of a power loss + // is in doubt. What that means is the caller's to decide. An import + // can count it as written (a lost entry only removes a key), a delete + // cannot (a lost removal brings a key back). + secure_fs::write_private_durably(&path, contents).map_err(|err| { + let path = path.display().to_string(); + if err.after_rename { + Error::Unflushed { + path, + source: err.source, + } + } else { + Error::Io { + path, + source: err.source, + } + } }) } diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index ebec73b1f..24fc99dfd 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -7,32 +7,26 @@ //! everything it knows about the chain arrives through the `BeaconNodeApi` //! trait, so it runs against any conformant node. //! -//! # Deviations from the specifications +//! # Slashing protection //! -//! This client implements **no slashing protection**. It keeps no durable -//! record of what it has signed, so a restart, or a second instance sharing the -//! same keystores, can double-vote or propose twice for one slot. On a live -//! network either is a slashable offence. This is a deliberate, recorded scope decision, not an oversight; -//! see [`docs/spec_deviations.md`] for the full entry, including what is and is -//! not covered. -//! -//! [`crate::attestation_guard`] and [`crate::proposal_guard`] close the part of -//! this that is reachable within a single run: a backward wall-clock step, or a -//! duty schedule replaced mid-epoch. Both are in-memory only and neither is a -//! substitute for the real thing. +//! Every block and attestation is checked against a durable, per-validator +//! record before it is signed, and the record is on disk before the signature +//! exists. The record keeps EIP-3076's minimal watermarks, lives next to the +//! keys, names the chain it belongs to, and is locked to one process. See +//! [`crate::slashing_protection`] for the rule and its guarantees, and +//! [`docs/spec_deviations.md`] for where the minimal rule is stricter than the +//! specification. //! //! [`docs/spec_deviations.md`]: https://github.com/lambdaclass/ethlambda/blob/main/docs/spec_deviations.md //! -//! The keymanager API is affected by the same decision. Its specification makes -//! `slashing_protection` a required field of the `DELETE /eth/v1/keystores` -//! response, so this implementation returns a well-formed but empty EIP-3076 -//! interchange, and accepts and ignores the optional `slashing_protection` -//! field on import. +//! History moves in and out as EIP-3076 interchange files: through the +//! keymanager API's `POST` and `DELETE /eth/v1/keystores`, and offline through +//! `ethlambda slashing-protection`. A key imported with no history is +//! registered as never having signed before it is loaded. pub mod aggregation; pub mod aggregation_selection; pub mod attestation; -pub mod attestation_guard; pub mod beacon_node; pub mod duties; pub mod error; @@ -41,10 +35,10 @@ pub mod keys; pub mod metrics; pub mod payload_attestation; pub mod proposal; -pub mod proposal_guard; pub mod proposer_settings; pub(crate) mod secure_fs; pub mod signing; +pub mod slashing_protection; pub mod slot_clock; pub mod subscriptions; pub mod sync_committee; @@ -95,29 +89,17 @@ pub struct ValidatorConfig { /// Bind address for the keymanager API. `None` when `--enable-keymanager` /// was not passed, in which case it is never spawned. pub keymanager: Option, + /// Register keys that have no slashing-protection record as never having + /// signed, instead of refusing to start. See + /// [`slashing_protection::SlashingProtection::open`]. + pub init_slashing_protection: bool, } /// Run the validator client until the process is stopped. pub async fn run(config: ValidatorConfig) -> Result<()> { metrics::init(); - // Said once, at startup, before anything is loaded or signed. - // - // Until now this client's most consequential property was recorded only in - // a rustdoc comment on this module, which nobody running a binary reads. - // An operator can build this, point it at mainnet keys, and never meet the - // warning. `warn!` rather than `info!` because the consequence is losing - // stake, and it is unconditional rather than behind a flag because a - // warning an operator can silence is one they will. - warn!( - "This validator client keeps NO slashing-protection record. It cannot detect that a \ - previous run, or another client holding these keys, already signed. Running the same \ - keys here and anywhere else, or restarting into a state where the chain has moved, can \ - produce a slashable double vote or a double block proposal. Do not use these keys in \ - any other client while this one runs. See docs/spec_deviations.md." - ); - - // Said at startup for the same reason the warning above is: an operator + // Said at startup, where an operator reads it: an operator // who never sets this proposes blocks that pay their execution-layer // rewards to whatever address their beacon node was configured with, which // on a default configuration is nobody useful. It is silent money, and @@ -135,6 +117,11 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { config.suggested_fee_recipient, )); + // The directory's lock first, before any key is read: held from here, + // nothing another process does to the directory can get between what + // this one loads and what it signs with. See `SlashingProtection::lock`. + let record_lock = slashing_protection::SlashingProtection::lock(&config.validators_dir)?; + let discovered = keys::definitions::ValidatorDefinitions::discover_if_absent( &config.validators_dir, &config.secrets_dir, @@ -192,17 +179,42 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { ); let clock = SlotClock::from_config(genesis.genesis_time, &spec); + + // After the beacon node names the chain, which the record must belong to, + // and before any service that signs exists. Every loaded key must already + // have a record, unless the operator said otherwise. + let keys = store.read().await.pubkeys(); + let protection = slashing_protection::SlashingProtection::open_locked( + record_lock, + &config.validators_dir, + genesis.genesis_validators_root, + &keys, + config.init_slashing_protection, + )?; + info!( + validators = protection.len(), + path = %protection.path().map(|path| path.display().to_string()).unwrap_or_default(), + "Slashing protection loaded" + ); + let protection = Arc::new(std::sync::Mutex::new(protection)); + let context = Arc::new(SigningContext { config: spec, genesis_validators_root: genesis.genesis_validators_root, }); let mut duties = DutiesService::new(beacon_node.clone(), Vec::new()); - let attestation = AttestationService::new(beacon_node.clone(), context.clone()); + let attestation = + AttestationService::new(beacon_node.clone(), context.clone(), protection.clone()); let aggregation = AggregationService::new(beacon_node.clone(), context.clone()); let payload_attestation = PayloadAttestationService::new(beacon_node.clone(), context.clone()); let sync_committee = SyncCommitteeService::new(beacon_node.clone(), context.clone()); - let proposal = ProposalService::new(beacon_node.clone(), context.clone(), settings.clone()); + let proposal = ProposalService::new( + beacon_node.clone(), + context.clone(), + settings.clone(), + protection.clone(), + ); if let Some(address) = config.keymanager { let token = http_api::load_or_create_token(&config.validators_dir)?; @@ -212,6 +224,8 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { validators_dir: config.validators_dir.clone(), secrets_dir: config.secrets_dir.clone(), definitions_lock: Arc::new(Mutex::new(())), + definitions_unflushed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + protection: protection.clone(), }; let router = http_api::router(context, token); let listener = tokio::net::TcpListener::bind(address) @@ -462,7 +476,7 @@ async fn attest_and_aggregate( // also delays the next slot's is worth less than nothing. // // Dropping the future mid-flight is safe here specifically because of - // `AttestationGuard`: it records at signing time, so a duty abandoned + // slashing protection: it records before signing, so a duty abandoned // between signing and submission cannot be re-signed next slot under // the same target. That is the intended outcome and matches `attest`'s // own "do not retry within a slot" rule; the attestation is simply @@ -620,8 +634,8 @@ async fn aggregate( /// So: a proposal that overruns is abandoned, and the slot's attesters still /// vote. Dropping the future mid-flight is safe for the same reason it is on /// the attestation path, and only for that reason: `ProposalService` records -/// the slot in its guard at signing time, so a block abandoned between signing -/// and publication cannot be signed a second time. +/// the slot in slashing protection before signing, so a block abandoned between +/// signing and publication cannot be signed a second time. /// /// Failures are logged and counted rather than propagated. A proposal is one /// slot's work; losing it must not stop the client attesting for the rest of @@ -1197,7 +1211,11 @@ mod tests { }]; serve_slot_duties( - &AttestationService::new(node.clone(), context.clone()), + &AttestationService::new( + node.clone(), + context.clone(), + slashing_protection::SlashingProtection::in_memory(), + ), &AggregationService::new(node.clone(), context.clone()), &PayloadAttestationService::new(node.clone(), context.clone()), &SyncCommitteeService::new(node.clone(), context.clone()), @@ -1335,7 +1353,11 @@ mod tests { }]; serve_slot_duties( - &AttestationService::new(node.clone(), context.clone()), + &AttestationService::new( + node.clone(), + context.clone(), + slashing_protection::SlashingProtection::in_memory(), + ), &AggregationService::new(node.clone(), context.clone()), &PayloadAttestationService::new(node.clone(), context.clone()), &SyncCommitteeService::new(node.clone(), context.clone()), @@ -1371,7 +1393,11 @@ mod tests { let store = empty_store(); serve_slot_duties( - &AttestationService::new(node.clone(), context.clone()), + &AttestationService::new( + node.clone(), + context.clone(), + slashing_protection::SlashingProtection::in_memory(), + ), &AggregationService::new(node.clone(), context.clone()), &PayloadAttestationService::new(node.clone(), context.clone()), &SyncCommitteeService::new(node.clone(), context.clone()), diff --git a/crates/validator/src/metrics.rs b/crates/validator/src/metrics.rs index 2da746bb3..ad52aa95b 100644 --- a/crates/validator/src/metrics.rs +++ b/crates/validator/src/metrics.rs @@ -74,19 +74,19 @@ static ATTESTATION_DEADLINE_MISSED_TOTAL: LazyLock = LazyLock::new(| .unwrap() }); -/// Attestations this process declined to sign because it had already signed a -/// conflicting one for that validator in this run. +/// Attestations this process declined to sign because slashing protection +/// judged them unsafe, or could not record them. /// /// Separate from `signing_failures_total`, which counts signatures that were /// attempted and failed. This counts signatures deliberately not attempted, and /// it should normally read zero: a non-zero value means the duty loop tried to -/// sign something the guard judged a double vote, which is worth investigating -/// even though the attestation was correctly suppressed. See -/// `crate::attestation_guard`. +/// sign something slashing protection judged a double or surround vote, which +/// is worth investigating even though the attestation was correctly +/// suppressed. See `crate::slashing_protection`. static ATTESTATIONS_REFUSED_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter!( "ethlambda_validator_attestations_refused_total", - "Attestations refused because this process already signed a conflicting one" + "Attestations refused by slashing protection" ) .unwrap() }); @@ -113,18 +113,18 @@ static BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL: LazyLock = LazyLock::new .unwrap() }); -/// Blocks this process declined to sign because it had already proposed that -/// slot for that validator in this run. +/// Blocks this process declined to sign because slashing protection judged +/// them unsafe, or could not record them. /// /// The proposal-shaped counterpart to `attestations_refused_total`, and it /// should read zero for the same reason: a non-zero value means the duty loop -/// tried to propose a slot the guard judged already proposed, which is worth -/// investigating even though the block was correctly suppressed. See -/// `crate::proposal_guard`. +/// tried to propose a slot slashing protection judged already proposed, which +/// is worth investigating even though the block was correctly suppressed. See +/// `crate::slashing_protection`. static BLOCKS_REFUSED_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter!( "ethlambda_validator_blocks_refused_total", - "Blocks refused because this process already proposed that slot" + "Blocks refused by slashing protection" ) .unwrap() }); @@ -248,6 +248,31 @@ static SIGNING_DURATION_SECONDS: LazyLock = LazyLock::new(|| { .unwrap() }); +/// Time spent durably writing the slashing-protection record: one write per +/// slot's attestation batch and one per block, each flushed to disk before +/// anything is signed. See `crate::slashing_protection`. +static SLASHING_PROTECTION_WRITE_SECONDS: LazyLock = LazyLock::new(|| { + register_histogram!( + "ethlambda_validator_slashing_protection_write_seconds", + "Time spent durably writing the slashing-protection record", + vec![ + 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0 + ] + ) + .unwrap() +}); + +/// Slashing-protection writes that failed, each of which left its messages +/// unsigned. Should read zero; anything else points at the disk under the +/// validators directory. +static SLASHING_PROTECTION_WRITE_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_slashing_protection_write_failures_total", + "Slashing-protection writes that failed, leaving their messages unsigned" + ) + .unwrap() +}); + static PUBLICATION_DELAY_SECONDS: LazyLock = LazyLock::new(|| { register_histogram!( "ethlambda_validator_publication_delay_seconds", @@ -334,6 +359,8 @@ pub fn init() { LazyLock::force(&BEACON_NODE_AVAILABLE); LazyLock::force(&SIGNING_DURATION_SECONDS); LazyLock::force(&PUBLICATION_DELAY_SECONDS); + LazyLock::force(&SLASHING_PROTECTION_WRITE_SECONDS); + LazyLock::force(&SLASHING_PROTECTION_WRITE_FAILURES_TOTAL); } pub fn set_validators_loaded(count: u64) { @@ -364,6 +391,10 @@ pub fn inc_attestations_refused() { ATTESTATIONS_REFUSED_TOTAL.inc(); } +pub fn inc_attestations_refused_by(count: u64) { + ATTESTATIONS_REFUSED_TOTAL.inc_by(count); +} + pub fn inc_attestation_deadline_missed() { ATTESTATION_DEADLINE_MISSED_TOTAL.inc(); } @@ -450,6 +481,14 @@ pub fn time_signing() -> TimingGuard { TimingGuard::new(&SIGNING_DURATION_SECONDS) } +pub fn time_slashing_protection_write() -> TimingGuard { + TimingGuard::new(&SLASHING_PROTECTION_WRITE_SECONDS) +} + +pub fn inc_slashing_protection_write_failures() { + SLASHING_PROTECTION_WRITE_FAILURES_TOTAL.inc(); +} + /// Record how long after a slot's start its attestations were accepted. pub fn observe_publication_delay(seconds: f64) { PUBLICATION_DELAY_SECONDS.observe(seconds); diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 1c76e4b37..c16319f07 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -9,13 +9,15 @@ //! back from the beacon node in between: //! //! 1. The **RANDAO reveal** for the slot's epoch, signed first because the node -//! cannot build a body without it. It is not slashable and is not guarded. +//! cannot build a body without it. It is not slashable and is not checked +//! against slashing protection. //! 2. The **block**, built by the beacon node around that reveal. This client //! never chooses a block's contents; it asks for one and checks that what //! came back is the one it asked for. -//! 3. The **block signature**, which is slashable and is guarded. +//! 3. The **block signature**, which is slashable, and which +//! [`crate::slashing_protection`] must allow and record first. //! -//! # Why the guard is consulted twice +//! # Why slashing protection is consulted twice //! //! Once before asking for a block, once before signing it. //! @@ -24,10 +26,11 @@ //! most expensive thing this client can ask of it. Discovering only afterwards //! that the slot was already proposed wastes that for nothing. //! -//! The check before signing is the one that matters, and it records. Recording -//! at signing time rather than after publication is what makes the duty loop's -//! deadline safe: a proposal abandoned between signing and publishing cannot be -//! re-signed, because the guard already counts that slot as proposed. +//! The check before signing is the one that matters, and it records, durably, +//! before the signature exists. Recording then rather than after publication is +//! what makes the duty loop's deadline and a crash both safe: a proposal +//! abandoned between signing and publishing cannot be re-signed, because the +//! record already counts that slot as proposed. //! //! # Gloas: the block, then its envelope //! @@ -37,7 +40,7 @@ //! builder domain, and it is published right after the block, in the same //! budget. A bid naming anyone else is a builder's to reveal, and is left alone. //! -//! The envelope is not slashable, so the guard is untouched by it. +//! The envelope is not slashable, so slashing protection is untouched by it. //! //! # Do not retry this call within a slot //! @@ -45,8 +48,8 @@ //! not be: a second call asks the node for a second block, which will differ //! from the first because the node has packed whatever arrived in between, and //! two distinct blocks for one slot from one validator is a slashable proposer -//! offence. The guard refuses it, so a retry is merely useless rather than -//! dangerous, but the caller should not be relying on the guard for that. +//! offence. Slashing protection refuses it, so a retry is merely useless rather +//! than dangerous, but the caller should not be relying on that. use std::sync::Arc; @@ -55,16 +58,16 @@ use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, ExecutionAddress, Root, Slot}; use ethlambda_types::beacon::signing::compute_epoch_at_slot; use tokio::sync::RwLock; -use tracing::{info, warn}; +use tracing::{error, info, warn}; use crate::beacon_node::block_contents::{GloasPayload, signed_envelope_ssz}; use crate::beacon_node::dto::{ProposerDutyDto, encode_hex, parse_pubkey}; use crate::beacon_node::{BeaconNodeApi, BlockRequest, Published, validate_produced_block}; use crate::error::{Error, Result}; use crate::keys::ValidatorStore; -use crate::proposal_guard::ProposalGuard; use crate::proposer_settings::ProposerSettings; use crate::signing::SigningContext; +use crate::slashing_protection::{Refusal, SharedProtection}; pub struct ProposalService { beacon_node: Arc, @@ -76,14 +79,9 @@ pub struct ProposalService { /// is the one that builds the payload, and the specification is explicit /// that it need not honour the preparation it was sent. settings: Arc, - /// What this process has already proposed, per validator. - /// - /// A `std::sync::Mutex` for the reason [`crate::attestation`]'s is: the - /// critical section is a lookup and an insert with no await inside it, so - /// an async mutex would buy nothing and cost a scheduling point. - /// - /// Not slashing protection. See [`ProposalGuard`]. - guard: std::sync::Mutex, + /// What every validator has signed, shared with the attestation service + /// and the keymanager. See [`crate::slashing_protection`]. + protection: SharedProtection, } impl ProposalService { @@ -91,12 +89,13 @@ impl ProposalService { beacon_node: Arc, context: Arc, settings: Arc, + protection: SharedProtection, ) -> Self { Self { beacon_node, context, settings, - guard: std::sync::Mutex::new(ProposalGuard::new()), + protection, } } @@ -163,37 +162,15 @@ impl ProposalService { let pubkey = parse_pubkey(&duty.pubkey)?; let epoch = compute_epoch_at_slot(slot); - // Before paying for a block, not after. See the module doc. - // - // A poisoned lock is treated as a refusal rather than unwrapped: the - // lock is poisoned only by a panic while it is held, nothing inside it - // can panic, and a crash in the duty path would be a worse outcome than - // a skipped proposal. - match self.guard.lock() { - Ok(guard) => { - if let Err(refusal) = guard.check(&pubkey, slot) { - warn!( - %slot, - validator = duty.validator_index, - %refusal, - "Refusing to propose: this process already proposed this slot" - ); - crate::metrics::inc_blocks_refused(); - return Err(Error::ProposalRefused { - slot, - reason: refusal.to_string(), - }); - } - } - Err(err) => { - warn!(%slot, %err, "Proposal guard is poisoned; refusing to propose"); - crate::metrics::inc_blocks_refused(); - return Err(Error::ProposalRefused { - slot, - reason: "the proposal guard is poisoned".to_string(), - }); - } - } + // Before paying for a block, not after. See the module doc. Off the + // executor even though it only reads: the record may be mid-write for + // another caller. A record that cannot be consulted is a refusal, not + // a crash in the duty path. + let checked = crate::slashing_protection::blocking(&self.protection, move |protection| { + Ok(protection.check_block(&pubkey, slot)) + }) + .await; + refuse_unless_safe(slot, duty.validator_index, "consult", checked)?; // Scoped away from every await, the way the attestation path's read // guard is, and for the reason spelled out there: a write-preferring @@ -225,11 +202,11 @@ impl ProposalService { // implementation that failed to honour its contract can be stopped // before it becomes a signature. // - // It matters more here than it reads. The guard records the *requested* - // slot while the signature covers the *produced* header's slot. If - // those ever diverged and both fell in one epoch, the domain would be - // identical and the result would be a valid, unguarded second block for - // a slot already proposed. + // It matters more here than it reads. Slashing protection records the + // *requested* slot while the signature covers the *produced* header's + // slot. If those ever diverged and both fell in one epoch, the domain + // would be identical and the result would be a valid, unrecorded + // second block for a slot already proposed. validate_produced_block(&request, &produced)?; // The node's fork and this client's must agree, or the signature is @@ -265,33 +242,22 @@ impl ProposalService { // The root is taken from the decoded container, never from anything // reassembled here, and it is taken once: the same value is what the - // guard's slot is recorded against and what the signature covers. + // recorded slot belongs to and what the signature covers. let block_root = produced.block_root(); + // The check that matters, and the one that records, durably, before + // the signature exists. Everything after this point may be abandoned + // without risk, because the record already counts this slot as + // proposed. Off the executor, and before the store's read lock below, + // so the disk flush never holds up a keymanager writer. + let recorded = crate::slashing_protection::blocking(&self.protection, move |protection| { + protection.check_and_record_block(&pubkey, slot) + }) + .await; + refuse_unless_safe(slot, duty.validator_index, "record", recorded)?; + let signature = { let store = store.read().await; - - // The check that matters, and the one that records. Everything - // after this point may be abandoned without risk, because the - // guard already counts this slot as proposed. - match self.guard.lock() { - Ok(mut guard) => guard.check_and_record(&pubkey, slot).map_err(|refusal| { - crate::metrics::inc_blocks_refused(); - Error::ProposalRefused { - slot, - reason: refusal.to_string(), - } - })?, - Err(err) => { - warn!(%slot, %err, "Proposal guard is poisoned; refusing to sign"); - crate::metrics::inc_blocks_refused(); - return Err(Error::ProposalRefused { - slot, - reason: "the proposal guard is poisoned".to_string(), - }); - } - } - self.context.sign_block(&store, &pubkey, block_root, slot)? }; @@ -436,11 +402,46 @@ impl ProposalService { } } +/// Slashing protection's answer about a block, as the duty's outcome: a judged +/// refusal, or a record that could not be consulted or written (`what` names +/// which), is a counted [`Error::ProposalRefused`]. +fn refuse_unless_safe( + slot: Slot, + validator: u64, + what: &'static str, + outcome: Result>, +) -> Result<()> { + let reason = match outcome { + Ok(Ok(())) => return Ok(()), + Ok(Err(refusal)) => { + warn!( + %slot, + validator, + %refusal, + "Refusing to propose: slashing protection judged this block unsafe" + ); + refusal.to_string() + } + Err(err) => { + error!( + %slot, + validator, + %err, + "Slashing protection could not {what} this block; refusing to sign" + ); + err.to_string() + } + }; + crate::metrics::inc_blocks_refused(); + Err(Error::ProposalRefused { slot, reason }) +} + #[cfg(test)] mod tests { use super::*; use crate::beacon_node::block_contents::SignedBlockContents; use crate::beacon_node::mock::MockBeaconNode; + use crate::slashing_protection::SlashingProtection; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::primitives::HashTreeRoot as _; use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, H160, Root}; @@ -477,7 +478,12 @@ mod tests { fn service(node: Arc) -> ProposalService { let settings = ProposerSettings::new(Bytes32::repeat_byte(0xab), None); - ProposalService::new(node, context(), Arc::new(settings)) + ProposalService::new( + node, + context(), + Arc::new(settings), + SlashingProtection::in_memory(), + ) } fn service_expecting( @@ -485,7 +491,12 @@ mod tests { fee_recipient: ExecutionAddress, ) -> ProposalService { let settings = ProposerSettings::new(Bytes32::repeat_byte(0xab), Some(fee_recipient)); - ProposalService::new(node, context(), Arc::new(settings)) + ProposalService::new( + node, + context(), + Arc::new(settings), + SlashingProtection::in_memory(), + ) } /// A slot inside mainnet's electra era. @@ -582,7 +593,12 @@ mod tests { let (store, pubkey) = store(); let node = Arc::new(MockBeaconNode::new().with_block(slot(), 7)); let settings = Arc::new(ProposerSettings::new(Bytes32::repeat_byte(0xab), None)); - let service = ProposalService::new(node.clone(), context(), settings.clone()); + let service = ProposalService::new( + node.clone(), + context(), + settings.clone(), + SlashingProtection::in_memory(), + ); settings.set_graffiti(&pubkey, Bytes32::repeat_byte(0xcd)); service @@ -628,7 +644,7 @@ mod tests { ); } - /// The guard is consulted before the block is asked for, not after, so a + /// Slashing protection is consulted before the block is asked for, not after, so a /// slot already proposed costs the beacon node nothing. #[tokio::test] async fn a_repeated_slot_is_refused_without_asking_for_a_block() { @@ -654,6 +670,65 @@ mod tests { assert_eq!(node.published_blocks().len(), 1); } + /// The restart shape: a fresh service over the same slashing-protection + /// directory refuses the slot already proposed, before asking for a block. + #[tokio::test] + async fn a_restart_does_not_forget_what_was_proposed() { + let (store, pubkey) = store(); + let dir = tempfile::tempdir().expect("temp dir"); + let settings = Arc::new(ProposerSettings::new(Bytes32::repeat_byte(0xab), None)); + let protection = || { + let opened = + SlashingProtection::open(dir.path(), Root::ZERO, &[pubkey], true).expect("opens"); + Arc::new(std::sync::Mutex::new(opened)) + }; + + let node = Arc::new(MockBeaconNode::new().with_block(slot(), 7)); + ProposalService::new(node.clone(), context(), settings.clone(), protection()) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("first"); + assert_eq!(node.published_blocks().len(), 1); + + let node_again = Arc::new(MockBeaconNode::new().with_block(slot(), 7)); + let err = ProposalService::new(node_again.clone(), context(), settings, protection()) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect_err("the slot was proposed before the restart"); + assert!(matches!(err, Error::ProposalRefused { .. }), "got {err:?}"); + assert!(node_again.block_requests().is_empty()); + assert!(node_again.published_blocks().is_empty()); + } + + /// A persisted record holding nothing for the key refuses it before a + /// block is asked for. The in-memory record of the other tests registers + /// a key on first use; a real one does not. + #[tokio::test] + async fn an_unregistered_key_is_refused_before_a_block_is_requested() { + let (store, pubkey) = store(); + let dir = tempfile::tempdir().expect("temp dir"); + let settings = Arc::new(ProposerSettings::new(Bytes32::repeat_byte(0xab), None)); + let opened = SlashingProtection::open(dir.path(), Root::ZERO, &[], false) + .expect("opens with no keys"); + let node = Arc::new(MockBeaconNode::new().with_block(slot(), 7)); + + let err = ProposalService::new( + node.clone(), + context(), + settings, + Arc::new(std::sync::Mutex::new(opened)), + ) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect_err("the key has no record"); + assert!( + matches!(&err, Error::ProposalRefused { reason, .. } if reason.contains("no slashing-protection record")), + "got {err:?}" + ); + assert!(node.block_requests().is_empty()); + assert!(node.published_blocks().is_empty()); + } + #[tokio::test] async fn a_node_that_cannot_import_the_block_is_reported_rather_than_hidden() { let (store, pubkey) = store(); @@ -745,12 +820,12 @@ mod tests { ); } - /// A failed production must leave the guard untouched, so the slot can - /// still be proposed if a later attempt succeeds. Recording on the early - /// check rather than at signing time would have burned it. + /// A failed production must leave slashing protection untouched, so the + /// slot can still be proposed if a later attempt succeeds. Recording on + /// the early check rather than at signing time would have burned it. /// /// The same service and the same node throughout, deliberately: a second - /// service would carry a second, empty guard and the test would pass + /// service would carry a second, empty record and the test would pass /// whether or not the first one recorded. #[tokio::test] async fn a_failed_production_does_not_burn_the_slot() { @@ -781,6 +856,7 @@ mod gloas_tests { use super::*; use crate::beacon_node::block_contents::SignedExecutionPayloadEnvelopeContents; use crate::beacon_node::mock::MockBeaconNode; + use crate::slashing_protection::SlashingProtection; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::gloas; use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; @@ -827,7 +903,12 @@ mod gloas_tests { ethlambda_types::beacon::primitives::Bytes32::repeat_byte(0xab), None, ); - ProposalService::new(node, context(), Arc::new(settings)) + ProposalService::new( + node, + context(), + Arc::new(settings), + SlashingProtection::in_memory(), + ) } fn verifies(pubkey: &BlsPubkey, signature: &[u8; 96], root: Root) -> bool { @@ -904,10 +985,15 @@ mod gloas_tests { )); let settings = Arc::new(ProposerSettings::new(Bytes32::repeat_byte(0xab), None)); settings.set_graffiti(&pubkey, Bytes32::repeat_byte(0xcd)); - ProposalService::new(node.clone(), context(), settings) - .propose(slot(), &duty(&pubkey), &store) - .await - .expect("proposes"); + ProposalService::new( + node.clone(), + context(), + settings, + SlashingProtection::in_memory(), + ) + .propose(slot(), &duty(&pubkey), &store) + .await + .expect("proposes"); let request = node.block_requests().remove(0); assert_eq!(request.fork, ForkName::Gloas); @@ -1006,8 +1092,9 @@ mod gloas_tests { assert!(node.published_envelopes().is_empty()); } - /// The guard's semantics are unchanged: a second proposal for the slot is - /// refused before asking for anything, and reveals no second envelope. + /// Slashing protection's semantics are unchanged at gloas: a second + /// proposal for the slot is refused before asking for anything, and + /// reveals no second envelope. #[tokio::test] async fn a_repeated_gloas_slot_is_refused_and_reveals_nothing_more() { let (store, pubkey) = store(); diff --git a/crates/validator/src/proposal_guard.rs b/crates/validator/src/proposal_guard.rs deleted file mode 100644 index 3d87c03a5..000000000 --- a/crates/validator/src/proposal_guard.rs +++ /dev/null @@ -1,271 +0,0 @@ -//! An in-process guard against signing two blocks for one slot. -//! -//! The proposal-shaped counterpart to [`crate::attestation_guard`], and the -//! same disclaimer applies in full: **this is not slashing protection.** It -//! holds nothing on disk, so it knows nothing about a previous run of this -//! process or about another process holding the same keys. A restart empties -//! it. -//! -//! # Why it is a separate guard rather than a field on the other one -//! -//! The two answer different questions and would answer them badly if merged. -//! An attestation is judged on its source and target *epochs*; a block is -//! judged on its *slot*, and a validator can legitimately propose in an epoch -//! it also attests in. Keying both off one record would either refuse a legal -//! proposal or admit an illegal one, depending on which field won. -//! -//! # The rule -//! -//! EIP-3076's minimal variant for blocks, per validator: remember the highest -//! slot proposed, and refuse anything that does not strictly advance it. -//! -//! One record per validator, like the attestation guard, and for the same -//! reason: the full condition ("never two distinct blocks for one slot") needs -//! the whole history to evaluate, while "the slot must strictly advance" needs -//! one number and is strictly more conservative. An honest proposer's blocks -//! already advance this way, since a validator is assigned at most one slot per -//! epoch and epochs only move forward. -//! -//! # What it actually closes -//! -//! The same two shapes the attestation guard closes, reached the same ways: -//! -//! - **A backward wall-clock step.** The duty loop takes its slot from -//! `SystemTime::now()`, which is not monotonic. An NTP correction can -//! re-enter a slot already proposed, and the block produced the second time -//! will differ from the first, because the beacon node has since seen more -//! attestations and a different head. Two distinct blocks for one slot from -//! one validator is a slashable proposer offence, and unlike a double vote it -//! needs no second validator to be caught: the two signed headers are the -//! whole evidence. -//! - **A proposer schedule replaced mid-epoch.** Proposer duties are only -//! final once the epoch's randao is fixed, so a refresh that lands late can -//! move a validator between slots within the epoch. Acting on both is two -//! blocks from one key. -//! -//! Refusing is always safe here, and cheaper than it is for an attestation: a -//! skipped proposal costs one block reward, and a proposer slashing is the most -//! expensive thing a validator can do. - -use std::collections::HashMap; - -use ethlambda_types::beacon::primitives::{BlsPubkey, Slot}; - -/// Why a proposal was refused. -/// -/// One variant, unlike [`crate::attestation_guard::Refusal`]'s two, because -/// there is only one rule to break. It stays an enum rather than a struct so -/// that the call site's `match`/`Display` shape matches the attestation path's, -/// and so a second rule can be added without changing the signature. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Refusal { - /// The slot is at or below one already proposed for this validator. - /// - /// Equality is refused as well as regression. The same slot proposed twice - /// is a slashable offence unless the two blocks are byte-identical, and - /// this guard deliberately does not keep enough to tell those apart. Nor - /// would it help: a block re-produced for the same slot is almost never - /// identical, since the beacon node packs whatever attestations have - /// arrived since. - SlotNotAdvanced { signed: Slot, proposed: Slot }, -} - -impl std::fmt::Display for Refusal { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::SlotNotAdvanced { signed, proposed } => write!( - f, - "slot {proposed} does not advance past {signed}, already proposed this run" - ), - } - } -} - -/// Per-validator record of the highest slot proposed in this run. -/// -/// Not `Clone`, for the reason [`crate::attestation_guard::AttestationGuard`] -/// is not: one guard per process is the point, and a copy would be a second -/// opinion about what has been signed. -#[derive(Debug, Default)] -pub struct ProposalGuard { - proposed: HashMap, -} - -impl ProposalGuard { - pub fn new() -> Self { - Self::default() - } - - /// Whether a block for `slot` may be signed for `pubkey`, recording - /// nothing. - /// - /// Separate from [`Self::record`] so a caller can decide before paying for - /// a block production round trip, which is far more expensive than the - /// attestation equivalent: it costs the beacon node an execution-layer - /// payload build. - pub fn check(&self, pubkey: &BlsPubkey, slot: Slot) -> Result<(), Refusal> { - let Some(&signed) = self.proposed.get(pubkey) else { - // Nothing proposed for this validator this run. As with the - // attestation guard, that says nothing about previous runs; the - // gap is the crate-level scope decision, not an oversight here. - return Ok(()); - }; - - if slot <= signed { - return Err(Refusal::SlotNotAdvanced { - signed, - proposed: slot, - }); - } - Ok(()) - } - - /// Record that a block for `slot` was signed for `pubkey`. - /// - /// Takes the maximum rather than overwriting, so a caller recording out of - /// order cannot lower the bar. Nothing does that today; making it - /// order-independent is one `max` and removes a class of bug that would - /// only ever appear under a clock step, which is the exact situation the - /// guard exists for. - pub fn record(&mut self, pubkey: BlsPubkey, slot: Slot) { - let entry = self.proposed.entry(pubkey).or_insert(slot); - *entry = (*entry).max(slot); - } - - /// Check and record in one step, for the common call site. - /// - /// Recording only on success is what makes a refusal idempotent: a refused - /// proposal must not move the bar it was measured against. - pub fn check_and_record(&mut self, pubkey: &BlsPubkey, slot: Slot) -> Result<(), Refusal> { - self.check(pubkey, slot)?; - self.record(*pubkey, slot); - Ok(()) - } - - /// How many validators have proposed something this run. For metrics. - pub fn len(&self) -> usize { - self.proposed.len() - } - - pub fn is_empty(&self) -> bool { - self.proposed.is_empty() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn pubkey(byte: u8) -> BlsPubkey { - BlsPubkey([byte; 48]) - } - - #[test] - fn a_first_proposal_is_allowed() { - let mut guard = ProposalGuard::new(); - assert!(guard.check_and_record(&pubkey(1), 96).is_ok()); - } - - #[test] - fn advancing_the_slot_is_allowed() { - let mut guard = ProposalGuard::new(); - guard.check_and_record(&pubkey(1), 96).expect("first"); - assert!(guard.check_and_record(&pubkey(1), 128).is_ok()); - } - - /// The backward-clock-step case: the same slot re-entered after an NTP - /// correction. The block produced the second time is a different block, - /// because the beacon node has packed whatever arrived in between, so this - /// is the proposer slashing the guard exists to stop. - #[test] - fn proposing_the_same_slot_twice_is_refused() { - let mut guard = ProposalGuard::new(); - guard.check_and_record(&pubkey(1), 96).expect("first"); - - let err = guard - .check_and_record(&pubkey(1), 96) - .expect_err("a second block for one slot must be refused"); - - assert_eq!( - err, - Refusal::SlotNotAdvanced { - signed: 96, - proposed: 96 - } - ); - } - - #[test] - fn a_regressed_slot_is_refused() { - let mut guard = ProposalGuard::new(); - guard.check_and_record(&pubkey(1), 128).expect("first"); - assert!(guard.check_and_record(&pubkey(1), 96).is_err()); - } - - /// The property that makes this per-validator rather than global: two of - /// this client's validators can be assigned different slots in one epoch, - /// and the second must not be blocked by the first. - #[test] - fn one_validator_does_not_block_another() { - let mut guard = ProposalGuard::new(); - guard - .check_and_record(&pubkey(1), 128) - .expect("first validator"); - - assert!( - guard.check_and_record(&pubkey(2), 96).is_ok(), - "a second validator proposing an earlier slot must be allowed" - ); - } - - #[test] - fn a_refusal_records_nothing() { - let mut guard = ProposalGuard::new(); - guard.check_and_record(&pubkey(1), 128).expect("first"); - guard - .check_and_record(&pubkey(1), 96) - .expect_err("regression refused"); - - // Still measured against 128, not against the refused 96: a proposal - // at 100 must still be refused. - assert!( - guard.check_and_record(&pubkey(1), 100).is_err(), - "the refused attempt must not have lowered the recorded slot" - ); - } - - #[test] - fn check_alone_does_not_record() { - let guard = ProposalGuard::new(); - guard.check(&pubkey(1), 96).expect("allowed"); - guard - .check(&pubkey(1), 96) - .expect("still allowed, because check records nothing"); - } - - #[test] - fn recording_out_of_order_does_not_lower_the_bar() { - let mut guard = ProposalGuard::new(); - guard.record(pubkey(1), 128); - guard.record(pubkey(1), 96); - - assert!( - guard.check(&pubkey(1), 100).is_err(), - "the earlier record must not have displaced the later one" - ); - } - - /// Slot 0 is a real slot, and `HashMap`'s absent-versus-zero distinction is - /// exactly the kind of thing a `unwrap_or_default()` would quietly erase. - /// A validator that proposed slot 0 must not be allowed to propose it - /// again. - #[test] - fn slot_zero_is_recorded_like_any_other() { - let mut guard = ProposalGuard::new(); - guard.check_and_record(&pubkey(1), 0).expect("first"); - assert!( - guard.check_and_record(&pubkey(1), 0).is_err(), - "slot 0 proposed twice must be refused, not treated as never proposed" - ); - } -} diff --git a/crates/validator/src/secure_fs.rs b/crates/validator/src/secure_fs.rs index 574056e9f..77a2b4ad9 100644 --- a/crates/validator/src/secure_fs.rs +++ b/crates/validator/src/secure_fs.rs @@ -1,6 +1,6 @@ //! Helpers for writing files that hold secrets or decide what a validator //! signs: the API token, imported keystores and their passwords, and the -//! validator definitions file. +//! validator definitions file, and the slashing-protection record. //! //! `std::fs::write` and `File::create` land at whatever mode the platform //! default gives them, `0644` under a standard umask on unix, which makes @@ -14,12 +14,12 @@ use std::path::Path; /// Write `contents`, creating the file if it does not exist and truncating it /// if it does, at mode `0600` on unix. /// -/// For files this crate is always the sole, trusted writer of and is willing -/// to overwrite outright: the API token, and the validator definitions file's -/// `.tmp` sibling (`ValidatorDefinitions::save` renames it into place, which -/// preserves the mode set here). Neither is named from attacker-influenced -/// input, so there is no symlink concern for these; see -/// [`write_private_no_symlink`] for the ones that are. +/// For a file this crate is the sole, trusted writer of and is willing to +/// overwrite outright: the API token. It is not named from +/// attacker-influenced input, so there is no symlink concern; see +/// [`write_private_no_symlink`] for the files that are, and +/// [`write_private_durably`] for the ones whose loss would change what a +/// validator signs. #[cfg(unix)] pub(crate) fn write_private(path: &Path, contents: impl AsRef<[u8]>) -> std::io::Result<()> { use std::io::Write as _; @@ -75,6 +75,154 @@ pub(crate) fn write_private_no_symlink( std::fs::write(path, contents) } +/// Why [`write_private_durably`] failed, and whether the file had already +/// been replaced when it did. +#[derive(Debug)] +pub(crate) struct DurableWriteError { + pub(crate) source: std::io::Error, + /// The rename had happened: the new file is at its path, and only the + /// directory flush that makes the rename survive a power loss failed. + /// Until the next successful flush the file on disk may be either one. + pub(crate) after_rename: bool, +} + +impl std::fmt::Display for DurableWriteError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.after_rename { + write!( + f, + "written, but the directory could not be flushed, so the write may not survive \ + a power loss: {}", + self.source + ) + } else { + self.source.fmt(f) + } + } +} + +impl std::error::Error for DurableWriteError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +/// Replace the file at `path` with `contents` so that a crash or power loss +/// leaves either the old file or the new one, never a mixture, and so that a +/// return of `Ok` means the new one is on disk. An error says whether the +/// file was already replaced when it failed (see [`DurableWriteError`]): +/// before the rename nothing at `path` changed; after it, only the flush that +/// makes the rename durable is missing. Unix only: elsewhere nothing is +/// touched and the write fails up front. +/// +/// Written to a sibling `.tmp` created fresh at mode `0600`, flushed, renamed +/// over `path`, and then the directory is flushed so the rename itself +/// survives. For files whose loss or truncation would change what a validator +/// signs: the slashing-protection record and the validator definitions file. +/// +/// The temporary is always a new file: whatever is at its path first, a +/// leftover of a write that crashed partway, or something planted there, is +/// removed, never reopened. A mode given at creation applies only at +/// creation, so reopening a leftover would carry its mode, whatever that was, +/// into the renamed record; and `O_EXCL` does not follow a symlink, so the +/// write cannot be redirected through one either. A write that fails before +/// the rename removes its temporary again, best effort. +#[cfg(unix)] +pub(crate) fn write_private_durably( + path: &Path, + contents: impl AsRef<[u8]>, +) -> Result<(), DurableWriteError> { + use std::io::Write as _; + let before = |source| DurableWriteError { + source, + after_rename: false, + }; + let mut name = path.file_name().unwrap_or_default().to_os_string(); + name.push(".tmp"); + let temporary = path.with_file_name(name); + + match std::fs::remove_file(&temporary) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(before(err)), + } + let staged = (|| { + let mut file = create_private(&temporary)?; + file.write_all(contents.as_ref())?; + file.sync_all() + })() + .and_then(|()| std::fs::rename(&temporary, path)); + if let Err(source) = staged { + let _ = std::fs::remove_file(&temporary); + return Err(before(source)); + } + // `Path::new("record.json").parent()` is `Some("")`, not `None`, and + // opening "" fails; the current directory is what it means. + let directory = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + sync_directory(directory).map_err(|source| DurableWriteError { + source, + after_rename: true, + }) +} + +/// Off unix the directory flush is not implemented, so the write is refused +/// before anything is touched rather than answer `Ok` for a rename that may +/// not have reached the disk. +#[cfg(not(unix))] +pub(crate) fn write_private_durably( + _path: &Path, + _contents: impl AsRef<[u8]>, +) -> Result<(), DurableWriteError> { + Err(DurableWriteError { + source: std::io::Error::new( + std::io::ErrorKind::Unsupported, + "durable writes are only implemented on unix", + ), + after_rename: false, + }) +} + +#[cfg(unix)] +fn create_private(path: &Path) -> std::io::Result { + use std::os::unix::fs::OpenOptionsExt as _; + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) +} + +/// Flush the directory, so a rename into it is itself durable. +/// +/// `File::sync_all` is `fsync` everywhere but macOS, where it is +/// `F_FULLFSYNC`, the only request there that reaches the disk rather than +/// the drive's cache, and the one the file's own flush above already relies +/// on. Some macOS filesystems refuse that request for a directory, so there a +/// plain `fsync` is tried next: it is the best those offer, and the original +/// error is kept if even that fails. Only there: on Linux the two are the +/// same call, and a second `fsync` after a failed one can answer success +/// while the kernel has already dropped the writeback error with the data, +/// which would make a flush that did not happen look durable. +#[cfg(unix)] +fn sync_directory(directory: &Path) -> std::io::Result<()> { + let directory = std::fs::File::open(directory)?; + let result = directory.sync_all(); + #[cfg(target_os = "macos")] + let result = result.or_else(|full_sync| { + use std::os::fd::AsRawFd as _; + // SAFETY: the descriptor is owned by `directory`, which outlives the call. + if unsafe { libc::fsync(directory.as_raw_fd()) } == 0 { + Ok(()) + } else { + Err(full_sync) + } + }); + result +} + #[cfg(all(test, unix))] mod tests { use super::*; @@ -94,6 +242,77 @@ mod tests { assert_eq!(std::fs::read_to_string(&path).expect("reads"), "shh"); } + #[test] + fn write_private_durably_replaces_the_file_at_mode_0600() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("record"); + write_private_durably(&path, "first").expect("writes"); + write_private_durably(&path, "second").expect("replaces"); + + assert_eq!(std::fs::read_to_string(&path).expect("reads"), "second"); + let mode = std::fs::metadata(&path) + .expect("metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600, "got {mode:o}"); + assert!( + !dir.path().join("record.tmp").exists(), + "the temporary file is renamed away" + ); + } + + #[test] + fn write_private_durably_never_inherits_a_leftover_temporary_file_mode() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("record"); + // A write that crashed partway, under a permissive umask. + let leftover = dir.path().join("record.tmp"); + std::fs::write(&leftover, "half").expect("writes leftover"); + std::fs::set_permissions(&leftover, std::fs::Permissions::from_mode(0o644)).expect("chmod"); + + write_private_durably(&path, "whole").expect("writes"); + + assert_eq!(std::fs::read_to_string(&path).expect("reads"), "whole"); + let mode = std::fs::metadata(&path) + .expect("metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600, "got {mode:o}"); + } + + #[test] + fn write_private_durably_does_not_write_through_a_symlink_at_the_temporary_path() { + let dir = tempfile::tempdir().expect("temp dir"); + let target = dir.path().join("target"); + std::fs::write(&target, "untouched").expect("writes target"); + let path = dir.path().join("record"); + std::os::unix::fs::symlink(&target, dir.path().join("record.tmp")).expect("symlinks"); + + write_private_durably(&path, "whole").expect("writes"); + + assert_eq!(std::fs::read_to_string(&path).expect("reads"), "whole"); + assert_eq!( + std::fs::read_to_string(&target).expect("reads"), + "untouched", + "the symlink target must be untouched" + ); + } + + /// A write that fails before the rename takes its temporary with it, and + /// says the file was not replaced. + #[test] + fn a_write_that_fails_before_the_rename_leaves_no_temporary() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("record"); + std::fs::create_dir(&path).expect("a directory in the file's place"); + + let err = write_private_durably(&path, "x").expect_err("cannot rename over a directory"); + + assert!(!err.after_rename, "{err}"); + assert!(!path.with_file_name("record.tmp").exists()); + assert!(path.is_dir(), "the directory is untouched"); + } + #[test] fn write_private_no_symlink_creates_a_mode_0600_file() { let dir = tempfile::tempdir().expect("temp dir"); diff --git a/crates/validator/src/signing.rs b/crates/validator/src/signing.rs index ced20d40e..9317ab341 100644 --- a/crates/validator/src/signing.rs +++ b/crates/validator/src/signing.rs @@ -367,8 +367,9 @@ impl SigningContext { /// This one *is* slashable, and is the most expensive signature this /// client can get wrong: two distinct blocks for one slot need no second /// validator to be caught, since the two signed headers are the whole - /// evidence. Callers must pass it through - /// [`crate::proposal_guard::ProposalGuard`] first. + /// evidence. Callers must have it allowed and recorded by + /// [`crate::slashing_protection::SlashingProtection::check_and_record_block`] + /// first. pub fn sign_block( &self, store: &ValidatorStore, diff --git a/crates/validator/src/slashing_protection/file.rs b/crates/validator/src/slashing_protection/file.rs new file mode 100644 index 000000000..865a8237c --- /dev/null +++ b/crates/validator/src/slashing_protection/file.rs @@ -0,0 +1,388 @@ +//! The slashing-protection file: its format, a strict reader, a durable +//! writer, and the lock that keeps a second process away from it. +//! +//! # Format +//! +//! The file is an EIP-3076 version 5 interchange in its minimal form: per +//! validator, at most one `signed_blocks` entry (the highest slot) and at most +//! one `signed_attestations` entry (the highest source and the highest target). +//! Storing the interchange format itself means the record a validator is +//! protected by and the record it would be exported with are the same bytes, +//! with no second encoding to drift from the first. +//! +//! # Reading fails closed +//! +//! Anything other than a well-formed file is an error, never an empty record. +//! An empty record would let every validator sign anything, so a file that is +//! empty, truncated, of another version, or names one validator twice must stop +//! the client rather than be taken as "nothing signed yet". Prysm's minimal +//! store reads an empty file as exactly that. +//! +//! # Writing is durable before it returns +//! +//! Every write goes to a temporary sibling, is flushed to disk, renamed over +//! the real file, and then the directory is flushed so the rename itself +//! survives a power loss. A reader therefore sees the old file or the new one, +//! never a mixture, and a write that returned `Ok` is on disk. + +use std::collections::HashMap; +use std::fs::File; +use std::path::Path; + +use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot}; +use ethlambda_types::beacon::serde_helpers::{hex_array, quoted_or_bare}; +use serde::{Deserialize, Serialize}; + +use super::{Record, Signed}; +use crate::error::{Error, Result}; + +/// The record's file name inside the validators directory. +pub const FILE_NAME: &str = "slashing_protection.json"; + +/// The only EIP-3076 interchange version this client reads or writes. +pub const INTERCHANGE_VERSION: &str = "5"; + +/// This client's own file, read strictly: a field it did not write is an +/// error, since nothing else should have written here. Not for reading +/// other clients' interchange files, which may carry fields this refuses; +/// an importer needs its own, lenient, types. +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Interchange { + metadata: Metadata, + data: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Metadata { + interchange_format_version: String, + #[serde(with = "hex_array")] + genesis_validators_root: [u8; 32], +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Entry { + #[serde(with = "hex_array")] + pubkey: [u8; 48], + signed_blocks: Vec, + signed_attestations: Vec, +} + +/// `signing_root` is optional in EIP-3076 and never written here: the minimal +/// rule does not compare roots. It is still accepted on read, so a file another +/// tool wrote in this shape is not rejected for carrying one. +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SignedBlock { + #[serde(with = "quoted_or_bare")] + slot: Slot, + #[serde(default, skip_serializing)] + #[allow(dead_code)] + signing_root: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SignedAttestation { + #[serde(with = "quoted_or_bare")] + source_epoch: Epoch, + #[serde(with = "quoted_or_bare")] + target_epoch: Epoch, + #[serde(default, skip_serializing)] + #[allow(dead_code)] + signing_root: Option, +} + +/// Parse an EIP-3076 version 5 interchange, complete or minimal, into its +/// chain and each entry's watermarks, in file order. +/// +/// A pubkey listed more than once is returned more than once: an import merges +/// them, while the record this client writes must never contain one twice. +/// The error is a reason, for the caller to attach to whatever it was reading. +pub(crate) fn parse(text: &str) -> std::result::Result<(Root, Vec<(BlsPubkey, Record)>), String> { + if text.trim().is_empty() { + return Err("empty; refusing to treat it as \"nothing signed\"".to_string()); + } + let interchange: Interchange = + serde_json::from_str(text).map_err(|err| format!("malformed: {err}"))?; + if interchange.metadata.interchange_format_version != INTERCHANGE_VERSION { + return Err(format!( + "interchange format version {:?}, expected {INTERCHANGE_VERSION:?}", + interchange.metadata.interchange_format_version + )); + } + let mut entries = Vec::with_capacity(interchange.data.len()); + for entry in &interchange.data { + let record = record_of(entry).map_err(|reason| { + format!( + "validator {}: {reason}", + crate::beacon_node::dto::encode_hex(&entry.pubkey) + ) + })?; + entries.push((BlsPubkey(entry.pubkey), record)); + } + Ok(( + Root::from(interchange.metadata.genesis_validators_root), + entries, + )) +} + +/// Read and validate the record at `path`. +pub(crate) fn read(path: &Path) -> Result<(Root, HashMap)> { + let invalid = |reason: String| Error::SlashingProtection { + path: path.display().to_string(), + reason, + }; + + let text = std::fs::read_to_string(path).map_err(|source| Error::Io { + path: path.display().to_string(), + source, + })?; + let (root, entries) = parse(&text).map_err(invalid)?; + + let mut records = HashMap::with_capacity(entries.len()); + for (pubkey, record) in entries { + if records.insert(pubkey, record).is_some() { + return Err(invalid(format!( + "validator {} appears more than once", + crate::beacon_node::dto::encode_hex(&pubkey.0) + ))); + } + } + Ok((root, records)) +} + +/// The watermarks one entry stands for: the highest slot, source and target it +/// lists. More than one entry of a kind is reduced rather than refused, since +/// taking the maximum is never less strict than any one of them. +/// +/// An attestation whose source is at or past its target is refused outside +/// genesis, the one shape the importer does not absorb: it is never valid, and +/// read into the record it would refuse every honest attestation after it +/// until the chain justified past that source. +fn record_of(entry: &Entry) -> std::result::Result { + let block_slot = entry.signed_blocks.iter().map(|block| block.slot).max(); + + let mut attestation: Option = None; + for signed in &entry.signed_attestations { + // The same shape the record and the attestation-data check refuse. + // Read into the record, it would refuse every honest attestation + // after it. + if super::is_impossible_source(signed.source_epoch, signed.target_epoch) { + return Err(format!( + "attestation source epoch {} is at or after its target epoch {}", + signed.source_epoch, signed.target_epoch + )); + } + let next = Signed { + source: signed.source_epoch, + target: signed.target_epoch, + }; + attestation = Some(match attestation { + Some(current) => current.max_with(next), + None => next, + }); + } + Ok(Record { + block_slot, + attestation, + }) +} + +/// Whether the record at `path` exists, failing closed. +/// +/// Only "no such file" is absence. Anything else that is not a regular file +/// (a symlink, perhaps to a volume that is not mounted, or a directory), and +/// any error reading its metadata, is an error: taking any of them for absence +/// would create a fresh record and orphan the real one. +pub(crate) fn exists(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_file() => Ok(true), + Ok(_) => Err(Error::SlashingProtection { + path: path.display().to_string(), + reason: "is not a regular file (a symlink or a directory); refusing to read or \ + replace it" + .to_string(), + }), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(source) => Err(Error::Io { + path: path.display().to_string(), + source, + }), + } +} + +/// `records` as a minimal EIP-3076 interchange for the chain named by +/// `genesis_validators_root`. +/// +/// Compact rather than pretty: the record is rewritten once per slot and per +/// block, and its size grows with the number of validators. Entries are +/// written in pubkey order so the bytes depend only on the content, which +/// keeps a diff of two files meaningful. +pub(crate) fn render<'a>( + genesis_validators_root: Root, + records: impl Iterator, +) -> serde_json::Result { + let mut data: Vec = records + .map(|(pubkey, record)| Entry { + pubkey: pubkey.0, + signed_blocks: record + .block_slot + .map(|slot| SignedBlock { + slot, + signing_root: None, + }) + .into_iter() + .collect(), + signed_attestations: record + .attestation + .map(|signed| SignedAttestation { + source_epoch: signed.source, + target_epoch: signed.target, + signing_root: None, + }) + .into_iter() + .collect(), + }) + .collect(); + data.sort_by_key(|entry| entry.pubkey); + + let interchange = Interchange { + metadata: Metadata { + interchange_format_version: INTERCHANGE_VERSION.to_string(), + genesis_validators_root: genesis_validators_root.0, + }, + data, + }; + serde_json::to_string(&interchange) +} + +/// Durably replace the record at `path` with `records`. +pub(crate) fn write<'a>( + path: &Path, + genesis_validators_root: Root, + records: impl Iterator, +) -> Result<()> { + let contents = + render(genesis_validators_root, records).map_err(|err| Error::SlashingProtection { + path: path.display().to_string(), + reason: format!("could not encode the record: {err}"), + })?; + crate::secure_fs::write_private_durably(path, contents).map_err(|err| { + let path = path.display().to_string(); + if err.after_rename { + Error::Unflushed { + path, + source: err.source, + } + } else { + Error::Io { + path, + source: err.source, + } + } + }) +} + +/// Take the exclusive lock in `directory`, held until the returned file is +/// dropped. +pub(crate) fn lock(directory: &Path) -> Result { + let io_error = |source| Error::Io { + path: directory.display().to_string(), + source, + }; + // The directory itself, not a lock file beside the record: the record is + // replaced by a rename on every write, so a lock on it would be on a + // stale inode; and a lock file can be deleted, after which a second + // process locks a fresh one and both sign. The directory is the one + // inode nothing here replaces, and `flock` takes a directory descriptor. + let file = File::open(directory).map_err(io_error)?; + match file.try_lock() { + Ok(()) => Ok(file), + Err(std::fs::TryLockError::WouldBlock) => Err(Error::SlashingProtection { + path: directory.display().to_string(), + reason: "in use by another process; two processes must never sign with the same keys" + .to_string(), + }), + Err(std::fs::TryLockError::Error(source)) => Err(io_error(source)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A source at its target, outside genesis, is refused on read as it is + /// on write: loaded, it would refuse every honest attestation after it. + #[test] + fn an_entry_with_a_source_at_its_target_is_refused() { + let entry = Entry { + pubkey: [1; 48], + signed_blocks: vec![], + signed_attestations: vec![SignedAttestation { + source_epoch: 5, + target_epoch: 5, + signing_root: None, + }], + }; + let err = record_of(&entry).expect_err("source at target"); + assert!(err.contains("at or after"), "{err}"); + + let genesis = Entry { + pubkey: [1; 48], + signed_blocks: vec![], + signed_attestations: vec![SignedAttestation { + source_epoch: 0, + target_epoch: 0, + signing_root: None, + }], + }; + assert!(record_of(&genesis).is_ok()); + } + + /// Several entries of a kind for one validator reduce to their maxima: + /// what keeps a file holding more than one from lowering a watermark. + #[test] + fn several_entries_of_a_kind_reduce_to_their_maxima() { + let entry = Entry { + pubkey: [1; 48], + signed_blocks: vec![ + SignedBlock { + slot: 7, + signing_root: None, + }, + SignedBlock { + slot: 3, + signing_root: None, + }, + ], + signed_attestations: vec![ + SignedAttestation { + source_epoch: 5, + target_epoch: 9, + signing_root: None, + }, + SignedAttestation { + source_epoch: 2, + target_epoch: 4, + signing_root: None, + }, + ], + }; + + let record = record_of(&entry).expect("valid"); + + assert_eq!( + record, + Record { + block_slot: Some(7), + attestation: Some(Signed { + source: 5, + target: 9 + }), + } + ); + } +} diff --git a/crates/validator/src/slashing_protection/mod.rs b/crates/validator/src/slashing_protection/mod.rs new file mode 100644 index 000000000..04f7c861d --- /dev/null +++ b/crates/validator/src/slashing_protection/mod.rs @@ -0,0 +1,1520 @@ +//! Slashing protection: the durable record of what each validator has signed. +//! +//! # The rule +//! +//! EIP-3076's minimal variant, per validator: +//! +//! - an attestation may be signed only if its target epoch is above the +//! highest one signed and its source epoch is not below the highest one +//! signed; +//! - a block may be signed only if its slot is above the highest one signed. +//! +//! That is stricter than the full slashing conditions, which would need the +//! whole signing history to evaluate: with one record per validator it cannot +//! tell "surrounds an attestation from six epochs ago" from "surrounds +//! nothing", and requiring monotonic progress makes the question unnecessary. +//! An honest validator's messages already advance this way. What the +//! strictness costs is a byte-identical re-sign after a crash, which is +//! refused; that is one missed attestation, against a slashing. +//! +//! # Order of operations +//! +//! Check, record durably, then sign. The record reaches the disk before the +//! signature exists, so a signature that escapes is always one the record +//! already accounts for, whether the process then crashes, the duty is +//! abandoned at its deadline, or the beacon node rejects it. A record that +//! cannot be written means nothing is signed. +//! +//! # What it guards against +//! +//! - **A restart.** The record outlives the process; see [`file`] for how it +//! is written and why reading it fails closed. +//! - **A second process on the same directory.** [`SlashingProtection::open`] +//! takes an exclusive lock in the validators directory for the life of the +//! process, so a second validator client pointed at that directory refuses +//! to start. The lock is per directory, not per key: the same keystore +//! copied into a second directory gets a second record and a second lock. +//! - **Another chain.** The record names the chain's `genesis_validators_root` +//! and is refused for any other. +//! - **A backward clock step or a duty schedule replaced mid-epoch**, the two +//! in-run shapes the in-memory guards this replaces were written for. +//! +//! # What it costs +//! +//! Every write rewrites the whole record, one compact JSON file, and flushes +//! it and its directory: the cost of a slot's attestation batch or a block +//! grows with the number of keys, not with the batch, and sits on the signing +//! path, off the executor. Fine for hundreds of keys on a local disk. For +//! thousands, or on a network volume whose flush takes tens of milliseconds +//! or more, `ethlambda_validator_slashing_protection_write_seconds` is the +//! number to watch, and the directory should be a local disk. +//! +//! It cannot see keys running in another client on another machine: no record +//! kept here can. A key moved here from elsewhere must bring its history with +//! it, which is why a key with no record is refused at startup unless the +//! operator says otherwise. A key registered with no history starts as never +//! having signed: the migration window, where the key may have signed +//! elsewhere a moment ago, is doppelganger protection's to close, not this +//! record's. + +pub mod file; + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use ethlambda_types::beacon::containers::shared::AttestationData; +use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot}; +use tracing::{debug, warn}; + +use crate::beacon_node::dto::encode_hex; +use crate::error::{Error, Result}; + +/// The handle the duty services and the keymanager share. +/// +/// A `std::sync::Mutex`: every critical section is a lookup, an insert and, on +/// a change, a file write, with no `.await` inside. Async callers go through +/// [`blocking`], so neither the write nor waiting for another holder's write +/// holds up the executor. +pub type SharedProtection = Arc>; + +/// How long [`blocking`] waits for the record before refusing, and how often +/// it looks. A flush takes milliseconds on a healthy disk; a quarter of a +/// 12-second slot is what a slow one gets. +const RECORD_WAIT: std::time::Duration = std::time::Duration::from_secs(3); +const RECORD_POLL: std::time::Duration = std::time::Duration::from_millis(5); + +/// Run `operation` on the record on tokio's blocking pool, and wait for it. +/// +/// Every async caller goes through here, including ones that only read: the +/// mutex can be held by another caller's write, a full rewrite and two +/// flushes, and waiting for it on an executor thread would stall whatever else +/// that thread runs. +/// +/// The operation's own error comes back as it is, so a caller can still tell +/// a write that failed (`Error::Io`, a disk to look at) from a record that +/// could not be reached at all (`Error::SlashingProtectionUnusable`): the +/// lock is poisoned, because an operation panicked while holding it, and +/// stays so until the process restarts, since nothing here recovers a record +/// that a panic may have left half-updated in memory. The panic that poisons +/// it is reported the same way on the call that raised it, which sees it as +/// the blocking task's `JoinError`. +/// +/// The operation runs to completion even if the caller is dropped, as a duty +/// abandoned at its deadline is: that is safe, since a record written for a +/// message that is then never signed only costs that message. +/// +/// A caller that finds the record held for [`RECORD_WAIT`] is refused, and +/// the duty is not retried within the slot: the holder is a flush still +/// running, and a retry would wait on that same flush. Nothing was recorded +/// for the refused duty, so the next slot's are unaffected; the write itself +/// is not bounded, since a flush cannot be interrupted, only outwaited. +pub async fn blocking( + protection: &SharedProtection, + operation: impl FnOnce(&mut SlashingProtection) -> Result + Send + 'static, +) -> Result { + let poisoned = || Error::SlashingProtectionUnusable { + reason: "an earlier access panicked while holding the record, so nothing is signed \ + until the validator client is restarted" + .to_string(), + }; + let protection = protection.clone(); + tokio::task::spawn_blocking(move || { + // A bounded wait rather than `lock()`: a write normally holds the + // record for milliseconds, so a wait this long means a flush is + // stuck, and refusing then keeps a stuck disk from piling up one + // blocking task per slot until the pool is full. Nothing is signed + // on a refusal. + let deadline = std::time::Instant::now() + RECORD_WAIT; + let mut protection = loop { + match protection.try_lock() { + Ok(guard) => break guard, + Err(std::sync::TryLockError::Poisoned(_)) => return Err(poisoned()), + Err(std::sync::TryLockError::WouldBlock) => { + if std::time::Instant::now() >= deadline { + return Err(Error::SlashingProtectionUnusable { + reason: format!( + "the record has been held by other accesses for {} s: a write \ + may be stuck, or the keymanager may be busy", + RECORD_WAIT.as_secs() + ), + }); + } + std::thread::sleep(RECORD_POLL); + } + } + }; + operation(&mut protection) + }) + .await + .map_err(|err| { + if err.is_panic() { + poisoned() + } else { + Error::SlashingProtectionUnusable { + reason: format!("the record could not be consulted: {err}"), + } + } + })? +} + +/// Whether an attestation's source epoch is at or past its target, which is +/// never valid outside genesis: with unsigned epochs, `(0, 0)` is the only +/// such pair that is. One predicate for the three places that refuse it, +/// the attestation-data check, the record and the file reader, so they +/// cannot drift. +pub(crate) fn is_impossible_source(source: Epoch, target: Epoch) -> bool { + source >= target && source != 0 +} + +/// The highest source and target epoch signed for one validator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Signed { + source: Epoch, + target: Epoch, +} + +impl Signed { + /// Each epoch's maximum, so that merging two records never lowers either + /// bar. + fn max_with(self, other: Signed) -> Signed { + Signed { + source: self.source.max(other.source), + target: self.target.max(other.target), + } + } +} + +/// An EIP-3076 interchange parsed once, for [`SlashingProtection::absorb`]: +/// the chain it names and each validator's watermarks, reduced as +/// [`SlashingProtection::import`] describes. +#[derive(Debug, Clone)] +pub struct StagedHistory { + root: Root, + entries: Vec<(BlsPubkey, Record)>, +} + +impl StagedHistory { + /// The chain the interchange names: what a record created for it must + /// belong to. + pub fn root(&self) -> Root { + self.root + } +} + +/// What [`SlashingProtection::absorb`] did. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Absorbed { + /// How many distinct validators the history named. + pub named: usize, + /// How many keys were newly registered as never having signed. + pub registered: usize, + /// The validators that already had a record and whose watermarks the + /// history raised: a caller running some of them can say so. + pub raised: Vec, +} + +/// What one validator has signed: its watermarks. +/// +/// `None` is "never signed", kept distinct from zero so that slot 0 and epoch +/// 0 stay signable once and only once. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Record { + block_slot: Option, + attestation: Option, +} + +impl Record { + fn check_block(&self, slot: Slot) -> std::result::Result<(), Refusal> { + match self.block_slot { + Some(signed) if slot <= signed => Err(Refusal::SlotNotAdvanced { + signed, + proposed: slot, + }), + _ => Ok(()), + } + } + + fn check_attestation(&self, data: &AttestationData) -> std::result::Result<(), Refusal> { + let (source, target) = (data.source.epoch, data.target.epoch); + // `validate_attestation_data` refuses the shape before it reaches the + // record; this is the record defending itself, since recording such a + // pair would refuse every honest attestation after it. + if is_impossible_source(source, target) { + return Err(Refusal::SourceAfterTarget { source, target }); + } + let Some(signed) = self.attestation else { + return Ok(()); + }; + if target <= signed.target { + return Err(Refusal::TargetNotAdvanced { + signed: signed.target, + proposed: target, + }); + } + if source < signed.source { + return Err(Refusal::SourceRegressed { + signed: signed.source, + proposed: source, + }); + } + Ok(()) + } + + fn with_block(self, slot: Slot) -> Record { + Record { + block_slot: Some(self.block_slot.map_or(slot, |signed| signed.max(slot))), + ..self + } + } + + /// Both records' maxima: what an import leaves when it meets an existing + /// record. Never less strict than either. + fn merged(self, other: Record) -> Record { + Record { + block_slot: self.block_slot.max(other.block_slot), + attestation: match (self.attestation, other.attestation) { + (Some(ours), Some(theirs)) => Some(ours.max_with(theirs)), + (ours, theirs) => ours.or(theirs), + }, + } + } + + fn with_attestation(self, data: &AttestationData) -> Record { + let next = Signed { + source: data.source.epoch, + target: data.target.epoch, + }; + Record { + attestation: Some( + self.attestation + .map_or(next, |signed| signed.max_with(next)), + ), + ..self + } + } +} + +/// Why a message was refused. +/// +/// Carried out to the caller so the log says which rule fired: a repeated +/// target usually means the clock moved or a schedule was replaced, while a +/// regressed source or an unregistered key means something stranger. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Refusal { + /// The key has no record, so nothing is known about what it signed. + /// + /// Not reachable through ordinary operation: startup refuses a key with no + /// record, and the keymanager registers a key before loading it. Refused + /// rather than signed if it is ever reached, since signing with no record + /// is exactly what this module exists to prevent. + Unregistered, + /// The attestation's target epoch is at or below one already signed. + /// + /// The double-vote shape. Equality is refused as well as regression: the + /// same target signed twice is a double vote unless the two attestations + /// are byte-identical, and the record deliberately does not keep enough to + /// tell those apart. + TargetNotAdvanced { signed: Epoch, proposed: Epoch }, + /// The attestation's source epoch is below one already signed. + /// + /// The surround shape: a lower source with a higher target surrounds the + /// earlier attestation. + SourceRegressed { signed: Epoch, proposed: Epoch }, + /// The attestation's source epoch is after its own target epoch. Never + /// valid, and never produced by an honest beacon node. + SourceAfterTarget { source: Epoch, target: Epoch }, + /// The block's slot is at or below one already signed. + /// + /// Equality is refused for the reason it is for an attestation's target, + /// and it would not help to allow it: a block re-produced for the same + /// slot is almost never identical, since the beacon node packs whatever + /// attestations have arrived since. + SlotNotAdvanced { signed: Slot, proposed: Slot }, +} + +impl std::fmt::Display for Refusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Unregistered => write!(f, "the validator has no slashing-protection record"), + Self::TargetNotAdvanced { signed, proposed } => write!( + f, + "target epoch {proposed} does not advance past {signed}, already signed" + ), + Self::SourceRegressed { signed, proposed } => write!( + f, + "source epoch {proposed} is below {signed}, already signed" + ), + Self::SourceAfterTarget { source, target } => { + write!( + f, + "source epoch {source} is at or after target epoch {target}" + ) + } + Self::SlotNotAdvanced { signed, proposed } => write!( + f, + "slot {proposed} does not advance past {signed}, already signed" + ), + } + } +} + +/// Where the record lives, and the lock that keeps it to this process. +#[derive(Debug)] +struct Storage { + path: PathBuf, + /// Held, never read: dropping it releases the lock. + _lock: std::fs::File, +} + +/// Every registered validator's watermarks, and where they are kept. +/// +/// Not `Clone`: one per process is the point. A copy would be a second opinion +/// about what has been signed, which is worse than none. +#[derive(Debug)] +pub struct SlashingProtection { + genesis_validators_root: Root, + records: HashMap, + /// `None` only for [`Self::in_memory`]. + storage: Option, +} + +impl SlashingProtection { + /// Open the record in `validators_dir` for the chain named by + /// `genesis_validators_root`, and hold its lock until the returned record is + /// dropped. + /// + /// Every key in `keys` must already have a record. A key without one is + /// refused unless `init` is set, in which case it is registered as never + /// having signed: the operator's statement that this key has signed + /// nothing the record should know about. A missing file is created only + /// on the same terms, or when there are no keys at all. + pub fn open( + validators_dir: &Path, + genesis_validators_root: Root, + keys: &[BlsPubkey], + init: bool, + ) -> Result { + let lock = Self::lock(validators_dir)?; + Self::open_locked(lock, validators_dir, genesis_validators_root, keys, init) + } + + /// Take the directory's exclusive lock ahead of [`Self::open_locked`], so + /// it can be held from before the keys are loaded: taken only at `open`, + /// a second process could load a key, wait for the first to delete it and + /// exit, then take the lock and sign with a key the file no longer lists. + pub fn lock(validators_dir: &Path) -> Result { + file::lock(validators_dir) + } + + /// [`Self::open`] with the lock already held, from [`Self::lock`]. + pub fn open_locked( + lock: std::fs::File, + validators_dir: &Path, + genesis_validators_root: Root, + keys: &[BlsPubkey], + init: bool, + ) -> Result { + let path = validators_dir.join(file::FILE_NAME); + let refused = |reason: String| Error::SlashingProtection { + path: path.display().to_string(), + reason, + }; + + let existed = file::exists(&path)?; + let mut records = if existed { + let (stored_root, records) = file::read(&path)?; + if stored_root != genesis_validators_root { + return Err(refused(format!( + "the record is for genesis_validators_root {}, but the beacon node is on {}; \ + these keys' history belongs to another chain", + encode_hex(&stored_root.0), + encode_hex(&genesis_validators_root.0) + ))); + } + records + } else { + HashMap::new() + }; + + let missing: Vec = keys + .iter() + .filter(|key| !records.contains_key(*key)) + .copied() + .collect(); + if !missing.is_empty() && !init { + let first = encode_hex(&missing[0].0); + return Err(refused(format!( + "{} validator key(s) have no slashing-protection record (first: {first}). \ + Start once with --init-slashing-protection only once they have signed \ + nothing anywhere for at least two epochs: stop the client that last ran \ + them and wait. That rules out a double vote, not a surround of an older \ + vote; a key with a history should bring it", + missing.len() + ))); + } + for key in &missing { + debug!( + pubkey = %encode_hex(&key.0), + "Registering a validator with no slashing-protection history" + ); + records.insert(*key, Record::default()); + } + if !missing.is_empty() { + warn!( + registered = missing.len(), + "Registered validators with no slashing-protection history \ + (--init-slashing-protection); each starts as never having signed" + ); + } + if init && missing.is_empty() && !keys.is_empty() { + warn!( + "--init-slashing-protection is set but every key already has a record; remove \ + the flag, or a keystore copied into the directory later is registered with no \ + history without anyone noticing" + ); + } + + // Written once whatever its state, as a probe: a filesystem that + // refuses to flush a directory (some FUSE, CIFS and overlay mounts + // answer EINVAL) would otherwise fail every slot's write, with the + // records advancing in memory and nothing ever signed, and nothing at + // startup to say why. Here it is a startup error naming the path. + file::write(&path, genesis_validators_root, records.iter())?; + Ok(Self { + genesis_validators_root, + records, + storage: Some(Storage { path, _lock: lock }), + }) + } + + /// Open the record in `validators_dir` without a beacon node, for the + /// `slashing-protection` command, holding its lock until the returned record + /// is dropped. + /// + /// The chain is whatever the record names. With no record yet, one is + /// started for `create_for` if given, which is how an interchange is + /// imported into a directory before the client first runs there; with + /// neither, this is an error. The file itself is only written by the + /// first change, so an import that fails, or imports nothing, leaves the + /// directory as it found it. A record created for the wrong chain is not + /// dangerous either: [`Self::open`] refuses it at the client's next start. + pub fn open_offline(validators_dir: &Path, create_for: Option) -> Result { + let lock = Self::lock(validators_dir)?; + let path = validators_dir.join(file::FILE_NAME); + let (genesis_validators_root, records) = if file::exists(&path)? { + file::read(&path)? + } else if let Some(root) = create_for { + (root, HashMap::new()) + } else { + return Err(Error::SlashingProtection { + path: path.display().to_string(), + reason: "there is no slashing-protection record in this directory".to_string(), + }); + }; + Ok(Self { + genesis_validators_root, + records, + storage: Some(Storage { path, _lock: lock }), + }) + } + + /// A record kept in memory only, with every key treated as registered on + /// first use. + /// + /// For tests, here and in other crates driving the duty services: it + /// protects within one process and nothing more, which is exactly what the + /// in-memory guards this module replaced did. The client itself only ever + /// uses [`Self::open`], and this is behind the `test-utils` feature so no + /// production caller can reach it. + #[cfg(any(test, feature = "test-utils"))] + pub fn in_memory() -> SharedProtection { + Arc::new(Mutex::new(Self { + genesis_validators_root: Root::ZERO, + records: HashMap::new(), + storage: None, + })) + } + + /// The record's path, or `None` for the in-memory record of the tests. + pub fn path(&self) -> Option<&Path> { + self.storage.as_ref().map(|storage| storage.path.as_path()) + } + + /// How many validators have a record. + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } + + fn record(&self, pubkey: &BlsPubkey) -> std::result::Result { + match self.records.get(pubkey) { + Some(record) => Ok(*record), + None if self.storage.is_none() => Ok(Record::default()), + None => Err(Refusal::Unregistered), + } + } + + /// Whether a block for `slot` may be signed for `pubkey`, recording + /// nothing. + /// + /// For deciding before paying for a block: production costs the beacon + /// node an execution payload build. Not a substitute for + /// [`Self::check_and_record_block`], which is the check that counts. + pub fn check_block(&self, pubkey: &BlsPubkey, slot: Slot) -> std::result::Result<(), Refusal> { + self.record(pubkey)?.check_block(slot) + } + + /// Check a block for `slot` and, if it may be signed, record it durably. + /// + /// The outer `Err` is a record that could not be written, and nothing may + /// be signed. Nothing changed either, with one exception: on + /// [`Error::Unflushed`] the slot is on disk and in memory (see + /// [`Self::commit`]), so a retry is refused as already proposed. Treat + /// every `Err` as "do not sign"; none of them means the old watermark + /// is still the one in force. + pub fn check_and_record_block( + &mut self, + pubkey: &BlsPubkey, + slot: Slot, + ) -> Result> { + let record = match self.record(pubkey) { + Ok(record) => record, + Err(refusal) => return Ok(Err(refusal)), + }; + if let Err(refusal) = record.check_block(slot) { + return Ok(Err(refusal)); + } + self.commit(HashMap::from([(*pubkey, record.with_block(slot))]))?; + Ok(Ok(())) + } + + /// Check one slot's attestations, all over `data`, and durably record every + /// one that may be signed, in a single write. + /// + /// The verdicts are in `pubkeys` order. A key listed twice is judged, at + /// its later position, against the verdict of its earlier one, so it is + /// refused the second time: same target as the one just staged. The outer + /// `Err` is a record that could not be written, and none of them may be + /// signed. Nothing changed either, with one exception: on + /// [`Error::Unflushed`] the target is on disk and in memory (see + /// [`Self::commit`]), so a retry is refused as already signed. Treat every + /// `Err` as "do not sign"; none of them means the old watermarks are + /// still the ones in force. + pub fn check_and_record_attestations( + &mut self, + pubkeys: &[BlsPubkey], + data: &AttestationData, + ) -> Result>> { + let mut staged: HashMap = HashMap::new(); + let verdicts = pubkeys + .iter() + .map(|pubkey| { + let record = match staged.get(pubkey) { + Some(record) => *record, + None => self.record(pubkey)?, + }; + record.check_attestation(data)?; + staged.insert(*pubkey, record.with_attestation(data)); + Ok(()) + }) + .collect(); + if !staged.is_empty() { + self.commit(staged)?; + } + Ok(verdicts) + } + + /// Register keys with no record as never having signed, and persist them + /// in one write. + /// + /// For keys added at runtime through the keymanager API, which loads a key + /// only after this returns. A key that already has a record keeps it: + /// registering never lowers a watermark. Returns how many were new. + pub fn register(&mut self, pubkeys: &[BlsPubkey]) -> Result { + self.absorb(None, pubkeys) + .map(|absorbed| absorbed.registered) + } + + /// The chain this record belongs to. + pub fn genesis_validators_root(&self) -> Root { + self.genesis_validators_root + } + + /// Whether `pubkey` has a record. + pub fn contains(&self, pubkey: &BlsPubkey) -> bool { + self.records.contains_key(pubkey) + } + + /// Every validator with a record. + pub fn pubkeys(&self) -> Vec { + self.records.keys().copied().collect() + } + + /// Merge an EIP-3076 version 5 interchange, complete or minimal, into the + /// record, in one durable write. + /// + /// Each validator in it is reduced to its highest slot, source and target + /// and merged with any existing record by taking the maximum of each, so an + /// import never lowers a watermark, and slashable data inside it (two + /// blocks for one slot, a surround) is absorbed rather than refused: the + /// result is at least as strict as every message the file lists. The one + /// shape refused is an attestation whose source is at or past its target + /// outside genesis: never valid, and read into the record it would refuse + /// every honest attestation until the chain justified past that source. + /// A validator it names that has no record yet is registered with the + /// imported watermarks. + /// + /// All or nothing: an interchange for another chain or of another version, + /// or one that does not parse, changes nothing. Returns how many distinct + /// validators it named. + pub fn import(&mut self, interchange: &str) -> Result { + let staged = Self::stage(interchange)?; + self.absorb(Some(&staged), &[]) + .map(|absorbed| absorbed.named) + } + + /// Parse `interchange` into what [`Self::absorb`] takes, off any lock: + /// its version and shape are judged here, its chain by `absorb`. + pub fn stage(interchange: &str) -> Result { + let (root, entries) = + file::parse(interchange).map_err(|reason| Error::SlashingProtection { + path: "interchange".to_string(), + reason, + })?; + Ok(StagedHistory { root, entries }) + } + + /// Whether `staged` is for this record's chain, changing nothing. + fn check_staged(&self, staged: &StagedHistory) -> Result<()> { + if staged.root != self.genesis_validators_root { + return Err(Error::SlashingProtection { + path: "interchange".to_string(), + reason: format!( + "it is for genesis_validators_root {}, but this record is for {}", + encode_hex(&staged.root.0), + encode_hex(&self.genesis_validators_root.0) + ), + }); + } + Ok(()) + } + + /// Merge `history`, if any, as [`Self::import`] describes, and register + /// every key in `register` that then still has no record, in one durable + /// write: either all of it is on disk or none of it is. Reports how many + /// validators the history named, how many keys were newly registered, + /// and which existing records the history raised. + /// + /// A key in `register` that already has a record and is not in the + /// history keeps that record, with a warning: it is what this client last + /// signed with the key, which says nothing about what the key signed + /// elsewhere since. Nothing is written when no watermark would change, + /// so absorbing a history twice costs one write, not two. + pub fn absorb( + &mut self, + history: Option<&StagedHistory>, + register: &[BlsPubkey], + ) -> Result { + let mut staged: HashMap = HashMap::new(); + if let Some(history) = history { + self.check_staged(history)?; + for (pubkey, imported) in &history.entries { + let current = staged + .get(pubkey) + .or_else(|| self.records.get(pubkey)) + .copied() + .unwrap_or_default(); + staged.insert(*pubkey, current.merged(*imported)); + } + } + let named = staged.len(); + let mut registered = 0; + for pubkey in register { + if staged.contains_key(pubkey) { + continue; + } + if self.records.contains_key(pubkey) { + warn!( + pubkey = %encode_hex(&pubkey.0), + "Registering a validator that already has a slashing-protection record and \ + brought no history; it continues from what this client last signed with it" + ); + continue; + } + staged.insert(*pubkey, Record::default()); + registered += 1; + } + staged.retain(|pubkey, record| self.records.get(pubkey) != Some(record)); + let raised = staged + .keys() + .filter(|pubkey| self.records.contains_key(*pubkey)) + .copied() + .collect(); + if !staged.is_empty() { + self.commit(staged)?; + } + Ok(Absorbed { + named, + registered, + raised, + }) + } + + /// The records of `pubkeys` as a minimal EIP-3076 version 5 interchange. + /// + /// A key with no record is left out rather than listed as never having + /// signed: an empty entry would tell the importer something this client + /// does not know. A key listed twice is exported once: an interchange that + /// names a validator twice is refused by stricter importers, this client's + /// own record reader among them. + pub fn export(&self, pubkeys: &[BlsPubkey]) -> Result { + let mut seen = HashSet::with_capacity(pubkeys.len()); + let selected = pubkeys + .iter() + .filter(|pubkey| seen.insert(**pubkey)) + .filter_map(|pubkey| self.records.get_key_value(pubkey)); + file::render(self.genesis_validators_root, selected).map_err(|err| { + Error::SlashingProtection { + path: "interchange".to_string(), + reason: format!("could not encode the export: {err}"), + } + }) + } + + /// Apply `changes` to the record: written to disk first, and taken into + /// memory only once the write has succeeded, so a failed write leaves both + /// as they were and nothing is signed. One failure is the exception: a + /// write that replaced the file but could not flush the directory + /// ([`Error::Unflushed`]) has the new watermarks on disk, perhaps only + /// until a power loss, and memory takes them too, so that what this + /// process holds is never weaker than what a reader of the file sees. + /// Nothing is signed on that path either. + fn commit(&mut self, changes: HashMap) -> Result<()> { + if let Some(storage) = &self.storage { + // What the record will hold, written without copying it: every + // unchanged entry, then the changed ones. + let next = self + .records + .iter() + .filter(|(pubkey, _)| !changes.contains_key(*pubkey)) + .chain(changes.iter()); + let _timing = crate::metrics::time_slashing_protection_write(); + if let Err(err) = file::write(&storage.path, self.genesis_validators_root, next) { + crate::metrics::inc_slashing_protection_write_failures(); + if matches!(err, Error::Unflushed { .. }) { + self.records.extend(changes); + } + return Err(err); + } + } + self.records.extend(changes); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use ethlambda_types::beacon::containers::shared::Checkpoint; + use ethlambda_types::beacon::primitives::H256; + + use super::*; + + fn pubkey(byte: u8) -> BlsPubkey { + BlsPubkey([byte; 48]) + } + + fn root(byte: u8) -> Root { + H256([byte; 32]) + } + + /// An attestation at `slot` voting source `source` and target `target`. + /// + /// `beacon_block_root` is a parameter because the double-vote case that + /// matters most is two attestations with one target epoch and *different* + /// roots, which is what a re-fetch after a block arrival produces. + fn data(slot: u64, source: Epoch, target: Epoch, block_root: u8) -> AttestationData { + AttestationData { + slot, + index: 0, + beacon_block_root: H256([block_root; 32]), + source: Checkpoint { + epoch: source, + root: Root::ZERO, + }, + target: Checkpoint { + epoch: target, + root: Root::ZERO, + }, + } + } + + /// One attestation, judged and recorded on its own. + fn attest( + protection: &mut SlashingProtection, + key: u8, + data: &AttestationData, + ) -> std::result::Result<(), Refusal> { + let mut verdicts = protection + .check_and_record_attestations(&[pubkey(key)], data) + .expect("the record is written"); + verdicts.pop().expect("one verdict") + } + + fn propose( + protection: &mut SlashingProtection, + key: u8, + slot: Slot, + ) -> std::result::Result<(), Refusal> { + protection + .check_and_record_block(&pubkey(key), slot) + .expect("the record is written") + } + + fn memory() -> SlashingProtection { + Arc::into_inner(SlashingProtection::in_memory()) + .expect("sole owner") + .into_inner() + .expect("not poisoned") + } + + fn open(dir: &Path, keys: &[u8], init: bool) -> Result { + let keys: Vec = keys.iter().map(|byte| pubkey(*byte)).collect(); + SlashingProtection::open(dir, root(0x42), &keys, init) + } + + /// Whether the permission-based failure tests can run: root bypasses + /// directory permissions, so as root the write they expect to fail + /// succeeds. Docker CI commonly runs as root. + #[cfg(unix)] + fn running_as_root() -> bool { + // SAFETY: `geteuid` reads a process attribute and cannot fail. + unsafe { libc::geteuid() == 0 } + } + + // The rule. Carried over from the in-memory guards this replaces. + + #[test] + fn a_first_attestation_is_allowed() { + assert!(attest(&mut memory(), 1, &data(96, 2, 3, 0xaa)).is_ok()); + } + + #[test] + fn advancing_the_target_is_allowed() { + let mut protection = memory(); + attest(&mut protection, 1, &data(96, 2, 3, 0xaa)).expect("first"); + assert!(attest(&mut protection, 1, &data(128, 3, 4, 0xbb)).is_ok()); + } + + /// The backward-clock-step case: same validator, same target epoch, a + /// different block root because the data was re-fetched after a block + /// arrived. + #[test] + fn re_attesting_one_target_with_different_data_is_refused() { + let mut protection = memory(); + attest(&mut protection, 1, &data(96, 2, 3, 0xaa)).expect("first"); + assert_eq!( + attest(&mut protection, 1, &data(96, 2, 3, 0xbb)), + Err(Refusal::TargetNotAdvanced { + signed: 3, + proposed: 3 + }) + ); + } + + /// The mid-epoch schedule replacement case: a different slot in the same + /// epoch is still one target epoch and still a double vote. + #[test] + fn a_different_slot_in_the_same_target_epoch_is_refused() { + let mut protection = memory(); + attest(&mut protection, 1, &data(96, 2, 3, 0xaa)).expect("first"); + assert!(attest(&mut protection, 1, &data(101, 2, 3, 0xbb)).is_err()); + } + + #[test] + fn a_regressed_target_is_refused() { + let mut protection = memory(); + attest(&mut protection, 1, &data(128, 3, 4, 0xaa)).expect("first"); + assert!(attest(&mut protection, 1, &data(96, 2, 3, 0xbb)).is_err()); + } + + /// The surround shape: a later target with an earlier source. + #[test] + fn a_regressed_source_under_an_advancing_target_is_refused() { + let mut protection = memory(); + attest(&mut protection, 1, &data(128, 3, 4, 0xaa)).expect("first"); + assert_eq!( + attest(&mut protection, 1, &data(160, 1, 5, 0xbb)), + Err(Refusal::SourceRegressed { + signed: 3, + proposed: 1 + }) + ); + } + + #[test] + fn a_source_after_its_target_is_refused() { + assert_eq!( + attest(&mut memory(), 1, &data(96, 4, 3, 0xaa)), + Err(Refusal::SourceAfterTarget { + source: 4, + target: 3 + }) + ); + } + + #[test] + fn one_validator_does_not_block_another_in_the_same_epoch() { + let mut protection = memory(); + attest(&mut protection, 1, &data(96, 2, 3, 0xaa)).expect("first validator"); + assert!(attest(&mut protection, 2, &data(101, 2, 3, 0xaa)).is_ok()); + } + + /// A refusal must not move the bar it was measured against. + #[test] + fn a_refusal_records_nothing() { + let mut protection = memory(); + attest(&mut protection, 1, &data(128, 3, 4, 0xaa)).expect("first"); + attest(&mut protection, 1, &data(160, 1, 5, 0xbb)).expect_err("surround refused"); + assert!( + attest(&mut protection, 1, &data(160, 3, 5, 0xcc)).is_ok(), + "the refused attempt must not have lowered the recorded source" + ); + } + + #[test] + fn a_key_twice_in_one_batch_is_refused_the_second_time() { + let mut protection = memory(); + let verdicts = protection + .check_and_record_attestations( + &[pubkey(1), pubkey(2), pubkey(1)], + &data(96, 2, 3, 0xaa), + ) + .expect("written"); + assert!(verdicts[0].is_ok()); + assert!(verdicts[1].is_ok()); + assert!(verdicts[2].is_err(), "the same vote twice in one batch"); + } + + #[test] + fn proposing_the_same_slot_twice_is_refused() { + let mut protection = memory(); + propose(&mut protection, 1, 96).expect("first"); + assert_eq!( + propose(&mut protection, 1, 96), + Err(Refusal::SlotNotAdvanced { + signed: 96, + proposed: 96 + }) + ); + } + + #[test] + fn a_regressed_slot_is_refused_and_another_validator_is_not() { + let mut protection = memory(); + propose(&mut protection, 1, 128).expect("first"); + assert!(propose(&mut protection, 1, 96).is_err()); + assert!(propose(&mut protection, 2, 96).is_ok()); + } + + #[test] + fn check_block_alone_records_nothing() { + let protection = memory(); + protection.check_block(&pubkey(1), 96).expect("allowed"); + protection + .check_block(&pubkey(1), 96) + .expect("still allowed, because check records nothing"); + } + + /// Slot 0 is a real slot: "never proposed" must not be spelled as zero. + #[test] + fn slot_zero_is_recorded_like_any_other() { + let mut protection = memory(); + propose(&mut protection, 1, 0).expect("first"); + assert!(propose(&mut protection, 1, 0).is_err()); + } + + /// A proposal and an attestation are judged on separate watermarks: a + /// validator can legitimately propose in an epoch it also attests in. + #[test] + fn a_proposal_does_not_block_an_attestation() { + let mut protection = memory(); + propose(&mut protection, 1, 100).expect("block"); + assert!(attest(&mut protection, 1, &data(100, 2, 3, 0xaa)).is_ok()); + } + + // Durability: the record outlives the process. + + #[test] + fn a_reopened_record_remembers_what_was_signed() { + let dir = tempfile::tempdir().expect("temp dir"); + { + let mut protection = open(dir.path(), &[1], true).expect("init"); + attest(&mut protection, 1, &data(96, 2, 3, 0xaa)).expect("attest"); + propose(&mut protection, 1, 97).expect("propose"); + } + + let mut protection = open(dir.path(), &[1], false).expect("reopen without init"); + assert!( + attest(&mut protection, 1, &data(96, 2, 3, 0xbb)).is_err(), + "the double vote from before the restart must be refused" + ); + assert!(propose(&mut protection, 1, 97).is_err()); + assert!(attest(&mut protection, 1, &data(128, 3, 4, 0xcc)).is_ok()); + } + + #[test] + fn a_key_with_no_record_is_refused_without_init() { + let dir = tempfile::tempdir().expect("temp dir"); + drop(open(dir.path(), &[1], true).expect("init")); + + let err = open(dir.path(), &[1, 2], false).expect_err("key 2 has no record"); + assert!( + err.to_string().contains("--init-slashing-protection"), + "{err}" + ); + + drop(open(dir.path(), &[1, 2], true).expect("init registers key 2")); + open(dir.path(), &[1, 2], false).expect("both now registered"); + } + + #[test] + fn a_missing_file_is_created_only_with_init_or_no_keys() { + let dir = tempfile::tempdir().expect("temp dir"); + open(dir.path(), &[1], false).expect_err("keys but no record"); + assert!(!dir.path().join(file::FILE_NAME).exists()); + + drop(open(dir.path(), &[], false).expect("no keys, nothing to protect yet")); + assert!(dir.path().join(file::FILE_NAME).exists()); + } + + #[test] + fn init_never_lowers_an_existing_record() { + let dir = tempfile::tempdir().expect("temp dir"); + { + let mut protection = open(dir.path(), &[1], true).expect("init"); + propose(&mut protection, 1, 97).expect("propose"); + } + let mut protection = open(dir.path(), &[1], true).expect("init again"); + assert!(propose(&mut protection, 1, 97).is_err()); + } + + #[test] + fn another_chain_is_refused() { + let dir = tempfile::tempdir().expect("temp dir"); + drop(open(dir.path(), &[1], true).expect("init")); + + let err = SlashingProtection::open(dir.path(), root(0x99), &[pubkey(1)], true) + .expect_err("another genesis_validators_root"); + assert!(err.to_string().contains("another chain"), "{err}"); + } + + #[test] + fn a_second_process_is_refused_while_the_first_holds_the_lock() { + let dir = tempfile::tempdir().expect("temp dir"); + let first = open(dir.path(), &[1], true).expect("first"); + + let err = open(dir.path(), &[1], false).expect_err("locked"); + assert!(err.to_string().contains("another process"), "{err}"); + + drop(first); + open(dir.path(), &[1], false).expect("free once the first is gone"); + } + + /// Every damaged shape is a startup error, never an empty record. + #[test] + fn a_damaged_file_fails_closed() { + let cases = [ + ("empty", String::new()), + ("whitespace", " \n".to_string()), + ("truncated", r#"{"metadata":{"interchange_format_version":"5","#.to_string()), + ("not json", "garbage".to_string()), + ( + "wrong version", + r#"{"metadata":{"interchange_format_version":"4","genesis_validators_root":"0x4242424242424242424242424242424242424242424242424242424242424242"},"data":[]}"#.to_string(), + ), + ]; + for (name, contents) in cases { + let dir = tempfile::tempdir().expect("temp dir"); + std::fs::write(dir.path().join(file::FILE_NAME), contents).expect("write"); + assert!( + open(dir.path(), &[], true).is_err(), + "{name} must fail closed" + ); + } + } + + #[test] + fn a_validator_listed_twice_fails_closed() { + let dir = tempfile::tempdir().expect("temp dir"); + let key = encode_hex(&pubkey(1).0); + let entry = format!(r#"{{"pubkey":"{key}","signed_blocks":[],"signed_attestations":[]}}"#); + let contents = format!( + r#"{{"metadata":{{"interchange_format_version":"5","genesis_validators_root":"{}"}},"data":[{entry},{entry}]}}"#, + encode_hex(&root(0x42).0) + ); + std::fs::write(dir.path().join(file::FILE_NAME), contents).expect("write"); + + let err = open(dir.path(), &[1], false).expect_err("duplicate"); + assert!(err.to_string().contains("more than once"), "{err}"); + } + + /// A write that fails changes nothing: not the file, and not what the + /// process believes was signed. + #[cfg(unix)] + #[test] + fn a_failed_write_refuses_and_changes_nothing() { + use std::os::unix::fs::PermissionsExt as _; + + if running_as_root() { + eprintln!("skipped: root bypasses directory permissions"); + return; + } + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[1], true).expect("init"); + attest(&mut protection, 1, &data(96, 2, 3, 0xaa)).expect("attest"); + + let read_only = std::fs::Permissions::from_mode(0o500); + std::fs::set_permissions(dir.path(), read_only).expect("chmod"); + let result = protection.check_and_record_attestations(&[pubkey(1)], &data(128, 3, 4, 0xbb)); + let writable = std::fs::Permissions::from_mode(0o700); + std::fs::set_permissions(dir.path(), writable).expect("chmod back"); + + assert!( + result.is_err(), + "the write must fail in a read-only directory" + ); + assert!( + attest(&mut protection, 1, &data(128, 3, 4, 0xbb)).is_ok(), + "the failed write must not have recorded target 4" + ); + } + + #[test] + fn the_file_is_an_eip_3076_minimal_interchange() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[1], true).expect("init"); + attest(&mut protection, 1, &data(96, 2, 3, 0xaa)).expect("attest"); + propose(&mut protection, 1, 97).expect("propose"); + + let text = std::fs::read_to_string(dir.path().join(file::FILE_NAME)).expect("read"); + let json: serde_json::Value = serde_json::from_str(&text).expect("json"); + assert_eq!(json["metadata"]["interchange_format_version"], "5"); + assert_eq!( + json["metadata"]["genesis_validators_root"], + encode_hex(&root(0x42).0) + ); + let entry = &json["data"][0]; + assert_eq!(entry["pubkey"], encode_hex(&pubkey(1).0)); + assert_eq!(entry["signed_blocks"][0]["slot"], "97"); + assert_eq!(entry["signed_attestations"][0]["source_epoch"], "2"); + assert_eq!(entry["signed_attestations"][0]["target_epoch"], "3"); + } + + /// The lock is on the directory, so a second opener of the same one is + /// refused, with nothing beside the record an operator could delete to + /// get around it. + #[test] + fn a_second_opener_of_the_same_directory_is_refused() { + let dir = tempfile::tempdir().expect("temp dir"); + let _first = open(dir.path(), &[1], true).expect("init"); + + let err = open(dir.path(), &[1], false).expect_err("locked"); + assert!( + err.to_string().contains("in use by another process"), + "{err}" + ); + assert!( + !dir.path().join("slashing_protection.lock").exists(), + "no lock file to delete" + ); + } + + #[test] + fn registering_keeps_an_existing_record() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[1], true).expect("init"); + propose(&mut protection, 1, 97).expect("propose"); + + assert_eq!( + protection + .register(&[pubkey(1), pubkey(2)]) + .expect("register"), + 1 + ); + assert!(propose(&mut protection, 1, 97).is_err()); + assert!(propose(&mut protection, 2, 97).is_ok()); + } + + /// A key registered with no history, at startup or at runtime, starts as + /// never having signed: slot 0 and epoch 0 are signable once. + #[test] + fn a_key_registered_with_no_history_starts_as_never_signed() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[1], true).expect("init"); + protection.register(&[pubkey(2)]).expect("register"); + + for key in [1, 2] { + assert!(propose(&mut protection, key, 0).is_ok(), "key {key}"); + assert!(propose(&mut protection, key, 0).is_err(), "key {key}"); + assert!( + attest(&mut protection, key, &data(0, 0, 0, 0xaa)).is_ok(), + "key {key}" + ); + assert!( + attest(&mut protection, key, &data(0, 0, 0, 0xbb)).is_err(), + "key {key}" + ); + } + } + + /// A symlink in the record's place, perhaps to a volume that is not + /// mounted, must not be taken for "no record" and replaced. + #[cfg(unix)] + #[test] + fn a_record_that_is_not_a_regular_file_fails_closed() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(file::FILE_NAME); + std::os::unix::fs::symlink(dir.path().join("unmounted/record.json"), &path) + .expect("symlinks"); + + let err = open(dir.path(), &[], true).expect_err("a dangling symlink"); + assert!(err.to_string().contains("not a regular file"), "{err}"); + assert!( + std::fs::symlink_metadata(&path) + .expect("still there") + .file_type() + .is_symlink(), + "the symlink must not have been replaced" + ); + } + + /// A minimal interchange for `key` on `root`. + fn interchange(root_byte: u8, key: u8, slot: Slot, source: Epoch, target: Epoch) -> String { + format!( + r#"{{"metadata":{{"interchange_format_version":"5","genesis_validators_root":"{}"}},"data":[{{"pubkey":"{}","signed_blocks":[{{"slot":"{slot}"}}],"signed_attestations":[{{"source_epoch":"{source}","target_epoch":"{target}"}}]}}]}}"#, + encode_hex(&root(root_byte).0), + encode_hex(&pubkey(key).0) + ) + } + + #[test] + fn an_import_raises_the_watermarks_and_survives_a_restart() { + let dir = tempfile::tempdir().expect("temp dir"); + { + let mut protection = open(dir.path(), &[], false).expect("opens"); + assert_eq!( + protection + .import(&interchange(0x42, 1, 97, 2, 3)) + .expect("imports"), + 1 + ); + } + let mut protection = open(dir.path(), &[1], false).expect("the imported key has a record"); + assert!(propose(&mut protection, 1, 97).is_err()); + assert!(attest(&mut protection, 1, &data(96, 2, 3, 0xaa)).is_err()); + assert!(attest(&mut protection, 1, &data(128, 3, 4, 0xaa)).is_ok()); + } + + /// A newer history imported over an older record raises it, and the raised + /// watermarks survive a restart: the merge updates an existing entry, not + /// only inserts a missing one. + #[test] + fn an_import_raises_an_existing_watermark() { + let dir = tempfile::tempdir().expect("temp dir"); + { + let mut protection = open(dir.path(), &[1], true).expect("init"); + propose(&mut protection, 1, 10).expect("propose"); + attest(&mut protection, 1, &data(96, 2, 3, 0xaa)).expect("attest"); + + protection + .import(&interchange(0x42, 1, 50, 4, 6)) + .expect("imports"); + assert!(propose(&mut protection, 1, 40).is_err()); + assert!(attest(&mut protection, 1, &data(160, 3, 5, 0xaa)).is_err()); + } + let mut protection = open(dir.path(), &[1], false).expect("reopens"); + assert!(propose(&mut protection, 1, 50).is_err()); + assert!(propose(&mut protection, 1, 51).is_ok()); + assert!(attest(&mut protection, 1, &data(192, 4, 6, 0xaa)).is_err()); + assert!(attest(&mut protection, 1, &data(224, 4, 7, 0xaa)).is_ok()); + } + + /// `absorb` says which existing records it raised, and only those: a + /// record the history creates, or leaves as it was, is not one. + #[test] + fn absorb_reports_the_records_it_raised() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[1, 2], true).expect("init"); + propose(&mut protection, 1, 10).expect("propose"); + propose(&mut protection, 2, 200).expect("propose"); + + let staged = SlashingProtection::stage(&interchange(0x42, 1, 50, 4, 6)).expect("stages"); + let absorbed = protection.absorb(Some(&staged), &[]).expect("absorbs"); + assert_eq!(absorbed.raised, vec![pubkey(1)]); + assert_eq!(absorbed.named, 1); + + // Already at these watermarks: nothing raised, nothing registered. + let again = protection + .absorb(Some(&staged), &[pubkey(1)]) + .expect("absorbs"); + assert_eq!( + again, + Absorbed { + named: 1, + ..Absorbed::default() + } + ); + + // A record the history creates is not a raised one. + let fresh = SlashingProtection::stage(&interchange(0x42, 3, 50, 4, 6)).expect("stages"); + let created = protection.absorb(Some(&fresh), &[]).expect("absorbs"); + assert!(created.raised.is_empty()); + assert!(protection.contains(&pubkey(3))); + } + + /// A source at or past its target is refused whole outside genesis, where + /// `(0, 0)` is what every validator's first attestation looks like. + #[test] + fn an_import_with_a_source_past_its_target_is_refused_whole() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[], false).expect("opens"); + let err = protection + .import(&interchange(0x42, 1, 97, 5, 2)) + .expect_err("source past target"); + assert!(err.to_string().contains("at or after"), "{err}"); + protection + .import(&interchange(0x42, 1, 97, 5, 5)) + .expect_err("source at target"); + assert!(protection.is_empty(), "nothing of a refused import is held"); + + protection + .import(&interchange(0x42, 1, 0, 0, 0)) + .expect("the genesis attestation imports"); + assert!(attest(&mut protection, 1, &data(32, 0, 1, 0xaa)).is_ok()); + } + + /// An older history imported over a newer record must not lower it. + #[test] + fn an_import_never_lowers_a_watermark() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[1], true).expect("init"); + propose(&mut protection, 1, 200).expect("propose"); + attest(&mut protection, 1, &data(320, 9, 10, 0xaa)).expect("attest"); + + protection + .import(&interchange(0x42, 1, 97, 2, 3)) + .expect("imports"); + assert!(propose(&mut protection, 1, 150).is_err()); + assert!(attest(&mut protection, 1, &data(160, 4, 5, 0xaa)).is_err()); + } + + #[test] + fn an_import_for_another_chain_changes_nothing() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[], false).expect("opens"); + let err = protection + .import(&interchange(0x99, 1, 97, 2, 3)) + .expect_err("another chain"); + assert!(err.to_string().contains("genesis_validators_root"), "{err}"); + assert!(protection.is_empty()); + } + + #[test] + fn an_export_lists_only_keys_with_a_record() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[1], true).expect("init"); + propose(&mut protection, 1, 97).expect("propose"); + + let exported = protection.export(&[pubkey(1), pubkey(2)]).expect("exports"); + let json: serde_json::Value = serde_json::from_str(&exported).expect("json"); + assert_eq!(json["data"].as_array().expect("array").len(), 1); + assert_eq!(json["data"][0]["signed_blocks"][0]["slot"], "97"); + + // And it imports back into a fresh record as the same watermarks. + let other = tempfile::tempdir().expect("temp dir"); + let mut fresh = open(other.path(), &[], false).expect("opens"); + fresh.import(&exported).expect("round trip"); + assert!(propose(&mut fresh, 1, 97).is_err()); + } + + /// A key requested twice is exported once: this client's own record + /// reader refuses a file naming a validator twice, and so may others'. + #[test] + fn an_export_lists_a_key_requested_twice_once() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[1], true).expect("init"); + propose(&mut protection, 1, 97).expect("propose"); + + let exported = protection.export(&[pubkey(1), pubkey(1)]).expect("exports"); + let json: serde_json::Value = serde_json::from_str(&exported).expect("json"); + assert_eq!(json["data"].as_array().expect("array").len(), 1); + + let other = tempfile::tempdir().expect("temp dir"); + let mut fresh = open(other.path(), &[], false).expect("opens"); + fresh.import(&exported).expect("round trip"); + assert!(propose(&mut fresh, 1, 97).is_err()); + } + + #[test] + fn opening_offline_needs_a_record_or_a_chain_to_create_one_for() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(file::FILE_NAME); + SlashingProtection::open_offline(dir.path(), None).expect_err("nothing to open"); + let mut created = + SlashingProtection::open_offline(dir.path(), Some(root(0x42))).expect("creates"); + assert_eq!(created.genesis_validators_root(), root(0x42)); + // Nothing is on disk until something is recorded, so an import that + // fails leaves no record behind for a chain nothing then runs on. + assert!(!path.exists(), "the record is not written at open"); + created + .import(&interchange(0x42, 1, 97, 2, 3)) + .expect("imports"); + assert!(path.exists(), "the first change writes the record"); + drop(created); + let reopened = SlashingProtection::open_offline(dir.path(), Some(root(0x99))) + .expect("an existing record keeps its own chain"); + assert_eq!(reopened.genesis_validators_root(), root(0x42)); + } + + /// Importing a history twice is the same as importing it once: the merge + /// takes maxima, so the second pass changes nothing. + #[test] + fn an_import_is_idempotent() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[], false).expect("opens"); + let history = interchange(0x42, 1, 97, 2, 3); + protection.import(&history).expect("imports"); + let once = protection.export(&[pubkey(1)]).expect("exports"); + assert_eq!(protection.import(&history).expect("imports again"), 1); + let twice = protection.export(&[pubkey(1)]).expect("exports"); + assert_eq!(once, twice); + assert!(propose(&mut protection, 1, 97).is_err()); + assert!(propose(&mut protection, 1, 98).is_ok()); + } + + #[test] + fn an_empty_interchange_is_refused() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[], false).expect("opens"); + for text in ["", "{}", "[]"] { + protection.import(text).expect_err("not an interchange"); + SlashingProtection::stage(text).expect_err("not an interchange"); + } + assert!(protection.is_empty()); + } + + #[test] + fn an_unregistered_key_is_refused_on_a_persisted_record() { + let dir = tempfile::tempdir().expect("temp dir"); + let mut protection = open(dir.path(), &[1], true).expect("init"); + assert_eq!(propose(&mut protection, 2, 97), Err(Refusal::Unregistered)); + assert_eq!( + attest(&mut protection, 2, &data(96, 2, 3, 0xaa)), + Err(Refusal::Unregistered) + ); + } +} diff --git a/crates/validator/tests/interchange_vectors.rs b/crates/validator/tests/interchange_vectors.rs new file mode 100644 index 000000000..a338d8b73 --- /dev/null +++ b/crates/validator/tests/interchange_vectors.rs @@ -0,0 +1,208 @@ +//! EIP-3076's shared interchange test vectors +//! (`eth-clients/slashing-protection-interchange-tests`), run against +//! `SlashingProtection`. +//! +//! Each case starts from an empty record, then for every step imports the +//! step's interchange and attempts its blocks and attestations, each of which +//! must be allowed or refused as the case says. This client keeps EIP-3076's +//! minimal watermarks, so the expectation read is `should_succeed`, the +//! minimal strategy's, rather than `should_succeed_complete`. +//! +//! The vectors are a download: `make slashing-protection-interchange-tests`, +//! which `make test` runs first. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use ethlambda_types::beacon::containers::shared::{AttestationData, Checkpoint}; +use ethlambda_types::beacon::primitives::{BlsPubkey, Root}; +use ethlambda_validator::beacon_node::dto::parse_pubkey; +use ethlambda_validator::slashing_protection::SlashingProtection; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Case { + name: String, + genesis_validators_root: String, + steps: Vec, +} + +#[derive(Deserialize)] +struct Step { + should_succeed: bool, + contains_slashable_data: bool, + interchange: serde_json::Value, + blocks: Vec, + attestations: Vec, +} + +#[derive(Deserialize)] +struct Block { + pubkey: String, + slot: String, + should_succeed: bool, +} + +#[derive(Deserialize)] +struct Attestation { + pubkey: String, + source_epoch: String, + target_epoch: String, + should_succeed: bool, +} + +fn vectors_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../slashing-protection-interchange-tests/tests/generated") +} + +#[test] +fn eip_3076_interchange_vectors() { + let dir = vectors_dir(); + assert!( + dir.is_dir(), + "{} is missing: run `make slashing-protection-interchange-tests`", + dir.display() + ); + + let mut paths: Vec = std::fs::read_dir(&dir) + .expect("reads the vectors directory") + .map(|entry| entry.expect("reads an entry").path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "json") + }) + .collect(); + paths.sort(); + assert!(!paths.is_empty(), "no vectors in {}", dir.display()); + + let failures: Vec = paths + .iter() + .filter_map(|path| { + let text = std::fs::read_to_string(path).expect("reads a vector"); + let case: Case = serde_json::from_str(&text).expect("parses a vector"); + run(&case) + .err() + .map(|reason| format!("{}: {reason}", case.name)) + }) + .collect(); + assert!( + failures.is_empty(), + "{} of {} vectors failed:\n{}", + failures.len(), + paths.len(), + failures.join("\n") + ); +} + +fn run(case: &Case) -> Result<(), String> { + let root = parse_root(&case.genesis_validators_root)?; + + // The vectors start from an empty database in which any validator may + // sign, so every key a case signs for is registered as never having signed: + // with no current slot, as before genesis. + let mut keys: Vec = Vec::new(); + let mut seen = HashSet::new(); + for step in &case.steps { + let texts = step.blocks.iter().map(|block| &block.pubkey).chain( + step.attestations + .iter() + .map(|attestation| &attestation.pubkey), + ); + for text in texts { + let pubkey = parse_pubkey(text).map_err(|err| err.to_string())?; + if seen.insert(pubkey.0) { + keys.push(pubkey); + } + } + } + + let dir = tempfile::tempdir().map_err(|err| err.to_string())?; + let mut protection = + SlashingProtection::open(dir.path(), root, &keys, true).map_err(|err| err.to_string())?; + + for (index, step) in case.steps.iter().enumerate() { + let imported = protection.import(&step.interchange.to_string()); + match (imported.is_ok(), step.should_succeed) { + (true, true) | (false, false) => {} + // The README lets an importer refuse an interchange holding + // slashable data, in which case the rest of the case is skipped. + // This one absorbs all of it but an attestation whose source is at + // or past its target, so a refusal on such a step ends the case, + // as it does in Lighthouse; on any other step it is a failure. + (false, true) if step.contains_slashable_data => return Ok(()), + (false, true) => { + return Err(format!( + "step {index}: import failed but should succeed (slashable data: {}): {}", + step.contains_slashable_data, + imported + .err() + .map(|err| err.to_string()) + .unwrap_or_default() + )); + } + (true, false) => { + return Err(format!("step {index}: import succeeded but should fail")); + } + } + + for (position, block) in step.blocks.iter().enumerate() { + let pubkey = parse_pubkey(&block.pubkey).map_err(|err| err.to_string())?; + let slot = parse_u64(&block.slot)?; + let allowed = protection + .check_and_record_block(&pubkey, slot) + .map_err(|err| err.to_string())? + .is_ok(); + if allowed != block.should_succeed { + return Err(format!( + "step {index}, block {position} at slot {slot}: allowed {allowed}, \ + expected {}", + block.should_succeed + )); + } + } + + for (position, attestation) in step.attestations.iter().enumerate() { + let pubkey = parse_pubkey(&attestation.pubkey).map_err(|err| err.to_string())?; + let source = parse_u64(&attestation.source_epoch)?; + let target = parse_u64(&attestation.target_epoch)?; + let verdicts = protection + .check_and_record_attestations(&[pubkey], &data(source, target)) + .map_err(|err| err.to_string())?; + let allowed = verdicts.first().is_some_and(|verdict| verdict.is_ok()); + if allowed != attestation.should_succeed { + return Err(format!( + "step {index}, attestation {position} ({source} -> {target}): allowed \ + {allowed}, expected {}", + attestation.should_succeed + )); + } + } + } + Ok(()) +} + +/// Only the epochs are judged; the rest of the data is filler. +fn data(source: u64, target: u64) -> AttestationData { + AttestationData { + slot: target * 32, + index: 0, + beacon_block_root: Root::ZERO, + source: Checkpoint { + epoch: source, + root: Root::ZERO, + }, + target: Checkpoint { + epoch: target, + root: Root::ZERO, + }, + } +} + +fn parse_u64(text: &str) -> Result { + text.parse().map_err(|err| format!("{text:?}: {err}")) +} + +fn parse_root(text: &str) -> Result { + ethlambda_validator::beacon_node::dto::parse_root(text).map_err(|err| err.to_string()) +} diff --git a/docs/cli.md b/docs/cli.md index fe7808e5b..7b3ce22ba 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -8,7 +8,7 @@ | `ethlambda beacon ` | The Ethereum Beacon Chain, as a gossip follower | | `ethlambda ` | `node`: the subcommand is injected | -Three subcommands follow no chain. `ethlambda replay-era` replays real chain +Four subcommands follow no chain. `ethlambda replay-era` replays real chain history from `.era` files and checks it against the states stored in them; see [`era_replay.md`](./era_replay.md) for its flags and checks. `ethlambda benchmark` is the offline benchmarking harness, covering block building (`benchmark synthetic`, run @@ -18,7 +18,9 @@ what each workload measures; `benchmark import`'s flags are in [their own section](#benchmark-import-flags) below. `ethlambda validator` is the validator client: a separate process that holds keys and performs duties against a beacon node over the standard REST API, documented in -[its own section](#validator-flags) below. +[its own section](#validator-flags) below. `ethlambda slashing-protection` +moves that client's EIP-3076 history in and out of its directory, offline; see +[its section](#slashing-protection) below. ## `node` is the default @@ -264,17 +266,24 @@ Both chains run a chain actor, so `RunningNode` carries a plain `BlockChain`. ## `validator` flags -`ethlambda validator` is not a node. It binds no libp2p port, opens no +`ethlambda validator` is not a node. It binds no libp2p port, opens no chain database, runs no discovery and follows no chain: it holds validator keys and performs duties against a beacon node's standard REST API. It works against any conformant beacon node, this repository's `beacon` subcommand or another implementation's, so none of the [common flags](#common-flags) apply to it. -> **It keeps no slashing-protection record.** Read -> [Spec Deviations](./spec_deviations.md#the-validator-client-keeps-no-slashing-protection-record) -> before running it with keys that hold real stake. In short: do not run these -> keys in any other client while this one runs, and treat a restart with the -> same care as a manual key move. The client repeats this warning at startup. +> **Slashing protection** lives in `--validators-dir/slashing_protection.json`, +> locked to one process, so `--validators-dir` must be writable: a read-only +> mount of the keys (a Kubernetes Secret, a `:ro` volume) fails at startup, as +> does a record that is a symlink (say, onto another volume) rather than a +> regular file, since a dangling one would read as "no record"; and +> the directory should be a local disk, since every signature waits for the +> record to be flushed to it, and the lock that keeps a second process off it +> is an `flock`, which a network filesystem may not honour. Unix only. A key with no record is refused at startup unless +> `--init-slashing-protection` is passed. A key moving here from another client +> brings its history with [`slashing-protection import`](#slashing-protection), +> or in the same keymanager API request that imports the key. See +> [Spec Deviations](./spec_deviations.md#the-validator-client-keeps-eip-3076s-minimal-watermarks-not-full-history). | Flag | Default | Meaning | |---|---|---| @@ -287,6 +296,7 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours | | `--graffiti` | empty | Text for the graffiti field of proposed blocks, for every validator the keymanager API gives no graffiti of its own. At most 32 bytes as UTF-8, right-padded with zeros. Refused rather than truncated if longer. An ethlambda beacon node appends both clients' versions to it (see [`rpc.md`](rpc.md#validator-endpoints)) | | `--enable-keymanager` | off | Serve the keymanager API: keystores, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | +| `--init-slashing-protection` | off | Register every loaded key with no slashing-protection record as never having signed, instead of refusing to start. Dangerous: only for keys that have signed nothing anywhere for at least two epochs, and even then it rules out a double vote only, not a surround of an older vote; a key with a history should bring it instead. Never lowers an existing record; remove it once the keys are registered, and startup warns while it is set for nothing | ### What it does today @@ -428,6 +438,14 @@ specification requires; `POST graffiti` refuses text over 32 bytes rather than truncating it; and every route answers `404` for a key this client does not hold. +The keystore routes carry slashing-protection history. `POST` refuses every +key in the request if its `slashing_protection` does not parse or is for +another chain, and otherwise writes that history before any key is saved, so a +key never loads ahead of it; a validator the history names without a keystore +in the request gets a record too. `DELETE` +returns the deleted keys' records as a minimal EIP-3076 interchange, answering +`not_active` for a key that is gone but still has a record. + The specification requires TLS. This binds to `--http-address`, loopback by default; exposing it further means putting a TLS terminator in front. @@ -439,6 +457,27 @@ beacon chain. Every series is registered at startup so it reads zero rather than being absent before its first event, which is what lets an alert fire on "attestations stopped". See [Metrics](./metrics.md#validator-client). +## `slashing-protection` + +Moves the validator client's EIP-3076 history in and out of its +`--validators-dir`, offline. It takes the lock the client holds while running, +so it refuses while a client runs on that directory, and it needs no beacon +node: the record names its own chain. + +```sh +# Bring a key's history from another client before its first start here. +ethlambda slashing-protection import --validators-dir ./validators interchange.json + +# Hand the history over before running the keys elsewhere. +ethlambda slashing-protection export --validators-dir ./validators out.json +ethlambda slashing-protection export --validators-dir ./validators --pubkeys 0xab..,0xcd.. out.json +``` + +| Sub-command | Does | +|---|---| +| `import ` | Merges a version 5 interchange, complete or minimal, into the record. Each validator is reduced to its highest slot, source and target and merged by maximum, so nothing already recorded is lowered and slashable data in the file is absorbed rather than refused, except an attestation whose source is at or past its target outside genesis, which is refused. All or nothing. With no record yet, the import creates one for the interchange's chain, and a failed import creates nothing; the client refuses the record at its next start if that is not its beacon node's chain | +| `export [--pubkeys ...] ` | Writes the record, or the named validators' part of it, as a minimal version 5 interchange, readable by its owner only. A named validator with no record fails the export, so a typo or the wrong directory cannot produce a file that reads as complete. Takes the record's lock, so the directory must be writable. Refuses to overwrite an existing file | + ## `benchmark import` flags Neither sub-command follows a chain; see [`benchmarking.md`](./benchmarking.md#import-workload) diff --git a/docs/introduction.md b/docs/introduction.md index 56451eb41..ca61c5182 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -45,7 +45,7 @@ into the rendered output: ## Community -- **Telegram**: [ethlambda group](https://t.me/lambdaconsensus), where we post daily updates; drop by to ask questions or chat about anything Lean-related. +- **Telegram**: [ethlambda group](https://t.me/ethlambda), where we post daily updates; drop by to ask questions or chat about anything Lean-related. - **X (Twitter)**: [@ethlambda_lean](https://twitter.com/ethlambda_lean) for occasional updates. - **Weekly community call**: every Friday, streamed live on [@class_lambda](https://x.com/class_lambda); the call link is posted on Telegram beforehand. - **Ecosystem coordination**: the [PQ Interop calls](https://github.com/ethereum/pm/issues?q=is%3Aissue+%22PQ+Interop%22+in%3Atitle) on `ethereum/pm` cover cross-client Lean Ethereum work and related updates; the meeting links are posted on each issue. diff --git a/docs/metrics.md b/docs/metrics.md index fe34487c6..10665c2a3 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -884,11 +884,11 @@ and absent is not the same as zero. | `ethlambda_validator_attestations_published_total` | Counter | Attestations a beacon node accepted | | `ethlambda_validator_attestation_failures_total` | Counter | Slots whose attestation duty failed and returned | | `ethlambda_validator_attestation_deadline_missed_total` | Counter | Slots whose duty never returned in time and was abandoned. Points at a slow or hung beacon node rather than a rejected attestation | -| `ethlambda_validator_attestations_refused_total` | Counter | Signatures deliberately not attempted, because this process had already signed a conflicting attestation for that validator. Should normally read zero | +| `ethlambda_validator_attestations_refused_total` | Counter | Signatures deliberately not attempted, because slashing protection judged the attestation unsafe or could not record it. Should normally read zero | | `ethlambda_validator_signing_failures_total` | Counter | Signatures attempted and failed | | `ethlambda_validator_blocks_proposed_total` | Counter | Blocks signed and accepted by a beacon node | | `ethlambda_validator_blocks_broadcast_not_imported_total` | Counter | Blocks a node broadcast but could not import into its own database, which is what a 202 means. A subset of `blocks_proposed_total`, not a failure: the block reached the network. Points at that node's execution layer | -| `ethlambda_validator_blocks_refused_total` | Counter | Blocks deliberately not signed, because this process had already proposed that slot for that validator. Should normally read zero | +| `ethlambda_validator_blocks_refused_total` | Counter | Blocks deliberately not signed, because slashing protection judged the block unsafe or could not record it. Should normally read zero | | `ethlambda_validator_block_proposal_failures_total` | Counter | Proposal duties that did not end in a published block, including ones abandoned for overrunning | | `ethlambda_validator_envelopes_published_total` | Counter | Execution payload envelopes a beacon node accepted for blocks this client proposed (gloas) | | `ethlambda_validator_envelope_failures_total` | Counter | Proposed gloas blocks whose self-built envelope was not published. The block is out and the proposal counts as made, but the slot's payload is withheld, so this is the failure that costs the payload. Should read zero | @@ -905,6 +905,8 @@ and absent is not the same as zero. | `ethlambda_validator_block_publication_delay_seconds` | Histogram | Slot start to block accepted. Its buckets are tighter than the attestation histogram's, because a block is due at the slot boundary rather than a third of the way in | | `ethlambda_validator_beacon_node_available` | Gauge | 1 when a beacon node answered the last duty refresh or attestation | | `ethlambda_validator_signing_duration_seconds` | Histogram | Time to sign one slot's batch, store read lock included | +| `ethlambda_validator_slashing_protection_write_seconds` | Histogram | Time to durably write the slashing-protection record, once per slot's attestation batch and once per block, before signing. Milliseconds on a local disk; worth an alert above about a second, since the duty waits for it and a write held past 3 s is refused | +| `ethlambda_validator_slashing_protection_write_failures_total` | Counter | Slashing-protection writes that failed, each leaving its messages unsigned. Should read zero; anything else points at the disk under `--validators-dir` | | `ethlambda_validator_publication_delay_seconds` | Histogram | Slot start to attestations accepted. The headline health number: it should sit near the one-third-slot duty offset | Three of these distinguish failures that look alike on a dashboard but have diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index bb6a5b32d..fd491cba5 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -40,58 +40,91 @@ target-slot order as they are scanned. - **Equivalence:** both produce a valid block. ethlambda front-loads the attestations that advance justification and finality, and within those tiers prefers the *newest* target where leanSpec takes the *oldest*; combined with the smaller default budget, an older entry can be outranked by newer ones round after round, so which votes reach peers through blocks differs even though every block stays valid. The smaller budget yields smaller blocks and lower build times. - **Upstream status:** the tiered strategy is proposed upstream as leanSpec [PR #1149](https://github.com/leanEthereum/leanSpec/pull/1149) (open at the time of writing), so this deviation may converge; the recursive-merge collapse follows leanSpec #510. -## The validator client keeps no slashing-protection record +## The validator client keeps EIP-3076's minimal watermarks, not full history -`ethlambda validator` signs attestations and blocks without recording what it -has signed. It cannot detect that a previous run, or another client holding the -same keys, already signed. +`ethlambda validator` records, per validator, only the highest block slot and +the highest attestation source and target epochs it has signed, and refuses +anything that does not advance past them. - **The specification:** [EIP-3076](https://eips.ethereum.org/EIPS/eip-3076) - defines an interchange format and the conditions a validator client must never - violate. For attestations: signing two *distinct* attestations with the same - target epoch (a double vote), and signing one whose source and target surround - or are surrounded by another's. For blocks: signing two distinct ones for the - same slot. Enforcing any of them requires a durable record, written before the - signature leaves the process. -- **ethlambda:** there is no such record and no store to hold one. This is a - deliberate scope decision for the client's first phase, not an oversight. -- **What partially covers it:** two in-memory guards, one per message kind. - `crate::attestation_guard::AttestationGuard` refuses to sign a second - conflicting attestation for a validator *within one run*, using EIP-3076's - minimal rule (the target must advance, the source must not regress). - `crate::proposal_guard::ProposalGuard` does the same for blocks, with the - minimal rule for those (the slot must strictly advance). Between them they - close the shapes reachable through ordinary operation: a backward wall-clock - step, or a duty schedule replaced mid-epoch. Both are held in memory only, so - a restart empties them and neither knows anything about any other process. -- **Gloas messages:** a gloas proposer also signs an execution payload - envelope, and a payload timeliness committee member signs a payload - attestation. Neither is slashable, so neither has a guard. An envelope is + forbids signing two *distinct* attestations with the same target epoch (a + double vote), one whose source and target surround or are surrounded by + another's, and two distinct blocks for one slot. It allows enforcing those + either against the full signing history or with the stricter minimal rule + over watermarks. +- **ethlambda:** the minimal rule. An attestation is signed only if its target + is above the recorded target and its source is not below the recorded source; + a block only if its slot is above the recorded slot. The record is + `/slashing_protection.json`, an EIP-3076 version 5 minimal + interchange naming the chain's `genesis_validators_root`. It is written to a + temporary file, flushed, renamed into place and the directory flushed, all + before the signature exists, and a write that fails leaves the message + unsigned. A file that is empty, malformed, of another version or for another + chain stops the client at startup rather than being read as "nothing + signed". An exclusive `flock` on the validators directory itself, the one + inode nothing replaces or deletes, keeps a second process away from it. See + `crate::slashing_protection`. +- **What the strictness costs:** a byte-identical re-sign is refused. A client + that recorded an attestation, crashed before publishing it, and restarted + into the same slot loses that attestation rather than publishing it; the same + holds for a block. An honest validator's messages otherwise advance this way, + so nothing else is refused. Lighthouse and Prysm keep signing roots to allow + the identical re-sign; Teku keeps the same watermarks as this client. +- **A wedged record:** the client refuses attestation data whose source is at + or past its target, which is never valid outside genesis, so an impossible + answer from a beacon node is refused rather than recorded. What it cannot + tell is a possible one: a source below the target but ahead of what the + chain has justified passes, the record keeps it, and every honest + attestation after it is refused as `SourceRegressed` until justification + reaches it, which during non-finality may be never. That is trust in the + beacon node, the same trust every attestation already places in it; the + symptom is the `SourceRegressed` refusal in the log and a non-zero + `attestations_refused_total`. There is no tool that lowers a watermark, by + design: stop the client, edit `slashing_protection.json` by hand, and start + it again. Lowering a watermark is only safe for a key that has not signed + past the value written. The same holds for a clock that jumps forward: the + target is pinned to the slot the local clock asked for, so a beacon node + that answers for a slot far ahead lets the record run ahead of the chain. + A bound against the clock would be circular; the guard is the beacon + node's own view of the head, which doppelganger protection will bring. +- **No checksum, and rewritten minimal at every start:** the record is an + EIP-3076 minimal interchange and nothing more, by design: readable by other + tools and editable by the operator, which the hand edit above relies on. A + checksum or a generation counter would make it this client's private format + and refuse that edit, to guard against a flipped digit that still parses, + which the disk's and the filesystem's own integrity checks are there for. + Every start rewrites the file in the minimal shape, so signing roots and + extra entries dropped in by hand collapse to their maxima. +- **Keys with no record:** refused at startup. `--init-slashing-protection` + registers them as never having signed, and a key imported through the + keymanager API with no history is registered the same way before it is + loaded. Nothing here stops such a key from signing again what it signed + elsewhere a moment ago, so the flag is only for keys that have signed nothing + anywhere for at least two epochs; the migration window is doppelganger + protection's to close, which this client does not have yet. +- **Messages outside the rule:** sync committee messages and contributions + are not slashable, and are not recorded. A gloas proposer also signs an + execution payload envelope, and a payload timeliness committee member signs + a payload attestation; neither is slashable, so neither is recorded. An envelope is published once per proposal, right after its block, and a failure to publish it is logged at error and counted without failing the proposal. A PTC vote has an in-memory dedup of `(validator, slot)` pairs recorded at signing time (`PayloadAttestationService`), so a duty abandoned between signing and - submission is lost rather than re-signed. It is emptied by a restart and says - nothing about another process, so it is not slashing protection either: a - second, different vote for one slot is equivocation that peers penalize in - gossip, not a slashing. -- **Why a proposer slashing is the worse of the two:** a double vote needs a - second validator's attestation to be caught. Two signed block headers for one - slot are the whole of the evidence on their own. -- **What is therefore not covered:** restarting into a state where the chain has - moved, and running these keys in two places at once. Both can produce a - slashable double vote or a double proposal, and nothing in this client will - stop them. -- **Keymanager consequences:** `DELETE /eth/v1/keystores` must return an - EIP-3076 interchange. This client returns a well-formed but empty one whose - `genesis_validators_root` is deliberately all-zero, so a conformant importer - rejects it outright rather than trusting an empty history that was never - recorded. `POST` accepts the `slashing_protection` field and ignores it, with - a server-side warning. A key migrated in or out through this API does not - carry its history. -- **Operationally:** do not run these keys in any other client while this one - runs, and treat a restart as an event that needs the same care a manual key - move would. The client warns about this at startup on every run. + submission is lost rather than re-signed. A second, different vote for one + slot is equivocation that peers penalize in gossip, not a slashing. +- **What is not covered:** the same keys running in another client on another + machine. No local record can see that. +- **Interchange:** an import (`ethlambda slashing-protection import`, or the + keymanager API's `POST /eth/v1/keystores`) reduces each validator to its + highest slot, source and target and merges them with the record by maximum, + so slashable data in the file is absorbed rather than refused, as Lighthouse + and Nimbus do. The one exception is an attestation whose source is at or + past its target outside genesis, which fails the whole import: it is never + valid, and read into the record it would refuse every honest attestation + until the chain justified past that source. An export (`ethlambda slashing-protection export`, or `DELETE + /eth/v1/keystores`) is the record itself, in the minimal format. Both pass + the shared `eth-clients/slashing-protection-interchange-tests` v5.3.0 vectors + under their `should_succeed` (minimal) expectations. ## Builder bids: gossip and API only (no builder API) diff --git a/tooling/kurtosis-assertoor/network_params.yaml b/tooling/kurtosis-assertoor/network_params.yaml index 03aec974b..bf96f4721 100644 --- a/tooling/kurtosis-assertoor/network_params.yaml +++ b/tooling/kurtosis-assertoor/network_params.yaml @@ -25,11 +25,15 @@ participants: cl_type: lighthouse vc_type: ethlambda vc_image: ghcr.io/lambdaclass/ethlambda:unstable + # Genesis keys in a fresh directory: nothing has ever signed with them, so + # registering them without history is safe. + vc_extra_params: ["--init-slashing-protection"] - el_type: geth cl_type: ethlambda cl_image: ghcr.io/lambdaclass/ethlambda:unstable vc_type: ethlambda vc_image: ghcr.io/lambdaclass/ethlambda:unstable + vc_extra_params: ["--init-slashing-protection"] network_params: # ethlambda only supports the mainnet preset: 32-slot epochs of 12 s. preset: mainnet