From d2e88790d75137063a00ebff86514190ff356c96 Mon Sep 17 00:00:00 2001 From: Zeljko Date: Thu, 19 Nov 2026 16:32:13 +0100 Subject: [PATCH 1/8] feat(server): startup account recovery from canonical delta history Add an opt-in startup check, GUARDIAN_STARTUP_ACCOUNT_RECOVERY, that runs before the API listeners open on a single Postgres-backed instance. It compares each Miden account's stored record with its latest canonical delta and with the configured node. When they disagree and the account's canonical deltas reach back to its creation, it rebuilds the account from them and stores the result once it matches the chain head and binds this server's ACK key. - The check holds a lease, runs within a 5 minute budget, saves where it stopped and continues on the next startup. It completes when one startup covers every account without deferring any. - The previous record is saved in account_recovery_backups. Accounts that were checked and not updated are listed in account_recovery_pending with the reason. - Guardian exits before opening its listeners when another process holds the recovery lease. Other recovery errors are logged and it serves. - infra: guardian_startup_account_recovery passes the flag and lengthens the task health check start period and the load balancer grace period. Docs: docs/PRODUCTION.md (Startup account recovery), docs/CONFIGURATION.md and infra/README.md. --- .../down.sql | 1 + .../up.sql | 8 + .../down.sql | 1 + .../up.sql | 6 + .../down.sql | 1 + .../up.sql | 11 + crates/server/src/builder/handle.rs | 44 + crates/server/src/builder/mod.rs | 7 + crates/server/src/coordination/mod.rs | 13 +- crates/server/src/jobs/account_recovery.rs | 618 ++++++++++ .../jobs/account_recovery/postgres_tests.rs | 1080 +++++++++++++++++ crates/server/src/jobs/mod.rs | 2 + crates/server/src/main.rs | 47 +- crates/server/src/schema.rs | 44 + crates/server/src/storage/postgres.rs | 2 + .../server/src/storage/postgres_recovery.rs | 526 ++++++++ docs/CONFIGURATION.md | 1 + docs/PRODUCTION.md | 98 ++ infra/README.md | 2 + infra/data.tf | 8 + infra/ecs.tf | 8 +- infra/terraform.tfvars.example | 1 + infra/variables.tf | 15 + 23 files changed, 2537 insertions(+), 7 deletions(-) create mode 100644 crates/server/migrations/2026-11-19-000001_account_recovery_backups/down.sql create mode 100644 crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql create mode 100644 crates/server/migrations/2026-11-19-000002_account_recovery_startup/down.sql create mode 100644 crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql create mode 100644 crates/server/migrations/2026-11-19-000003_account_recovery_pending/down.sql create mode 100644 crates/server/migrations/2026-11-19-000003_account_recovery_pending/up.sql create mode 100644 crates/server/src/jobs/account_recovery.rs create mode 100644 crates/server/src/jobs/account_recovery/postgres_tests.rs create mode 100644 crates/server/src/storage/postgres_recovery.rs diff --git a/crates/server/migrations/2026-11-19-000001_account_recovery_backups/down.sql b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/down.sql new file mode 100644 index 00000000..2f06cdd1 --- /dev/null +++ b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/down.sql @@ -0,0 +1 @@ +DROP TABLE account_recovery_backups; diff --git a/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql new file mode 100644 index 00000000..eeae9278 --- /dev/null +++ b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql @@ -0,0 +1,8 @@ +CREATE TABLE account_recovery_backups ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + account_id VARCHAR(128) NOT NULL, + prior_state JSONB NOT NULL, + prior_auth JSONB NOT NULL, + recovered_commitment VARCHAR(128) NOT NULL, + recovered_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/crates/server/migrations/2026-11-19-000002_account_recovery_startup/down.sql b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/down.sql new file mode 100644 index 00000000..4b4d93cc --- /dev/null +++ b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/down.sql @@ -0,0 +1 @@ +DROP TABLE account_recovery_startup; diff --git a/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql new file mode 100644 index 00000000..51c4f0ec --- /dev/null +++ b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql @@ -0,0 +1,6 @@ +CREATE TABLE account_recovery_startup ( + network_config JSONB PRIMARY KEY, + resume_after VARCHAR(128) NOT NULL DEFAULT '', + completed_at TIMESTAMPTZ, + unresolved_accounts BIGINT +); diff --git a/crates/server/migrations/2026-11-19-000003_account_recovery_pending/down.sql b/crates/server/migrations/2026-11-19-000003_account_recovery_pending/down.sql new file mode 100644 index 00000000..bdb06264 --- /dev/null +++ b/crates/server/migrations/2026-11-19-000003_account_recovery_pending/down.sql @@ -0,0 +1 @@ +DROP TABLE account_recovery_pending; diff --git a/crates/server/migrations/2026-11-19-000003_account_recovery_pending/up.sql b/crates/server/migrations/2026-11-19-000003_account_recovery_pending/up.sql new file mode 100644 index 00000000..426c0ffe --- /dev/null +++ b/crates/server/migrations/2026-11-19-000003_account_recovery_pending/up.sql @@ -0,0 +1,11 @@ +CREATE TABLE account_recovery_pending ( + account_id VARCHAR(128) PRIMARY KEY, + outcome TEXT NOT NULL CHECK (outcome IN ('deferred', 'unresolved')), + reason TEXT NOT NULL, + stored_commitment VARCHAR(128) NOT NULL, + canonical_nonce BIGINT, + canonical_commitment VARCHAR(128), + history_starts_at_nonce BIGINT, + history_base_commitment VARCHAR(128), + recorded_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/crates/server/src/builder/handle.rs b/crates/server/src/builder/handle.rs index a486739e..913b6da6 100644 --- a/crates/server/src/builder/handle.rs +++ b/crates/server/src/builder/handle.rs @@ -54,6 +54,11 @@ pub struct ServerHandle { pub(crate) stats_leader: std::sync::Arc, /// Single-owner lease for the chain-driven release sweep (issue #434). pub(crate) release_sweep_leader: std::sync::Arc, + #[cfg(feature = "postgres")] + pub(crate) account_recovery: Option<( + std::sync::Arc, + crate::metadata::MidenNetworkType, + )>, pub(crate) startup_info: StartupInfo, pub(crate) cors_layer: Option, pub(crate) rate_limit_config: Option, @@ -65,7 +70,46 @@ pub struct ServerHandle { pub(crate) grpc_port: u16, } +#[cfg(feature = "postgres")] +#[derive(Debug, thiserror::Error)] +pub enum StartupRecoveryError { + #[error("Another process already holds the account recovery lease")] + LeaseBusy, + #[error("Could not establish exclusive account recovery: {0}")] + LeaseUnavailable(String), + #[error("{0}")] + Incomplete(String), +} + +#[cfg(feature = "postgres")] +impl From for StartupRecoveryError { + fn from(reason: String) -> Self { + Self::Incomplete(reason) + } +} + impl ServerHandle { + /// Recover accounts once before a stop-before-start deployment opens its listeners. + #[cfg(feature = "postgres")] + pub async fn recover_accounts_on_startup(&self) -> Result<(), StartupRecoveryError> { + if crate::middleware::rate_limit::max_replicas_from_env()? != 1 { + tracing::warn!( + "Skipping startup account recovery because GUARDIAN_MAX_REPLICAS exceeds 1" + ); + return Ok(()); + } + let (store, network_type) = self + .account_recovery + .as_ref() + .ok_or_else(|| "Account recovery requires shared Postgres storage".to_owned())?; + if self.app_state.canonicalization.is_none() { + return Err(StartupRecoveryError::Incomplete( + "Account recovery requires canonicalization".into(), + )); + } + crate::jobs::account_recovery::run_once(&self.app_state, store, *network_type).await + } + /// Run the server with the configured settings pub async fn run(self) { self.startup_info.log(); diff --git a/crates/server/src/builder/mod.rs b/crates/server/src/builder/mod.rs index 4891c869..9a4c67d4 100644 --- a/crates/server/src/builder/mod.rs +++ b/crates/server/src/builder/mod.rs @@ -506,6 +506,11 @@ impl ServerBuilder { let ack = self.ack.ok_or("AckRegistry not set. Use .ack(...)")?; let coordination = self.coordination; + #[cfg(feature = "postgres")] + let account_recovery = coordination + .as_ref() + .and_then(|handles| handles.account_recovery.clone()) + .map(|store| (store, crate::metadata::MidenNetworkType::from(network_type))); // Fail closed before anything else: the Postgres backend must never fall // back to per-process coordination (AlwaysLeader + in-memory sessions), // which would let every replica run canonicalization and split auth @@ -790,6 +795,8 @@ impl ServerBuilder { leader, stats_leader, release_sweep_leader, + #[cfg(feature = "postgres")] + account_recovery, startup_info, cors_layer: self.cors_layer, rate_limit_config: Some(rate_limit_config), diff --git a/crates/server/src/coordination/mod.rs b/crates/server/src/coordination/mod.rs index 236a7dff..b4311f5a 100644 --- a/crates/server/src/coordination/mod.rs +++ b/crates/server/src/coordination/mod.rs @@ -72,6 +72,8 @@ pub struct CoordinationHandles { pub stats_leader: Arc, /// Single-owner lease for the chain-driven release sweep (issue #434). pub release_sweep_leader: Arc, + #[cfg(feature = "postgres")] + pub(crate) account_recovery: Option>, /// Account-scoped leases that fence Guardian executions (issue #254). pub execution_leases: Arc, /// Shared publication store for the `/dashboard/stats` aggregate. @@ -112,6 +114,8 @@ impl CoordinationHandles { RELEASE_SWEEP_LEASE, "single-process", )), + #[cfg(feature = "postgres")] + account_recovery: None, execution_leases: Arc::new(InMemoryExecutionLeases::new()), stats_store: Arc::new(InMemoryStatsStore::new()), block_watch_store: Arc::new(InMemoryBlockWatchStore::new()), @@ -155,7 +159,14 @@ impl CoordinationHandles { release_sweep_leader: Arc::new(PgLeaseElector::new( pool.clone(), RELEASE_SWEEP_LEASE, - holder_id, + holder_id.clone(), + )), + account_recovery: Some(Arc::new( + crate::storage::postgres::recovery::RecoveryStore::new( + pool.clone(), + cipher.clone(), + holder_id, + ), )), execution_leases: Arc::new(PgExecutionLeases::new(pool.clone())), stats_store: Arc::new(PgStatsStore::new(pool.clone(), cipher)), diff --git a/crates/server/src/jobs/account_recovery.rs b/crates/server/src/jobs/account_recovery.rs new file mode 100644 index 00000000..654ae276 --- /dev/null +++ b/crates/server/src/jobs/account_recovery.rs @@ -0,0 +1,618 @@ +use std::{fmt::Display, future::Future, time::Duration}; + +use guardian_shared::SignatureScheme; +use tokio::time::{Instant, timeout, timeout_at}; +use tokio_util::sync::CancellationToken; + +use crate::builder::handle::StartupRecoveryError; +use crate::coordination::Lease; +use crate::delta_object::DeltaObject; +use crate::jobs::canonicalization::spawn_lease_renewal; +use crate::metadata::{Auth, MidenNetworkType}; +use crate::network::{ + AppliedState, NetworkClient, ReconstructError, RpcReadMode, StateVerification, reconstructor, +}; +use crate::state::AppState; +use crate::storage::is_record_corruption; +use crate::storage::postgres::recovery::{ + PendingOutcome, RecoveryCandidate, RecoveryStore, ScanProgress, +}; + +const PAGE_SIZE: i64 = 100; +const MAX_DELTAS: u32 = 128; +const LEASE_TTL: Duration = Duration::from_secs(30); +const SCAN_BUDGET: Duration = Duration::from_secs(300); +const REMAINING_COUNT_BUDGET: Duration = Duration::from_secs(1); +const SCAN_CONTROL_BUDGET: Duration = Duration::from_secs(6); +const STALE_LEASE_WAIT: Duration = Duration::from_secs(35); +const LEASE_RETRY_INTERVAL: Duration = Duration::from_secs(2); +const MAX_CONSECUTIVE_FAILED_CHECKS: usize = 3; + +#[derive(Debug)] +enum RecoveryOutcome { + Unchanged, + Verified(Box, Auth), + Retryable(String), + Unresolved(String), +} + +enum PendingRecord { + Clear, + Write(PendingOutcome, String), +} + +#[derive(Debug, Default)] +struct ScanSummary { + checked: usize, + restored: usize, + unchanged: usize, + retryable: usize, + unresolved: i64, + failed: usize, + last_checked: Option, + stopped_early: bool, +} + +#[derive(Debug, PartialEq, Eq)] +enum ScanStep { + Completed { unresolved: i64 }, + Resume(ScanProgress), +} + +impl ScanSummary { + /// What a scan that started at `started` leaves for the next startup: its position when + /// it stopped early, and otherwise a pass from the first account. Only a pass that + /// started at the first account, reached the last one within one startup and deferred + /// none completes the scan. + fn next_step(&self, started: ScanProgress) -> ScanStep { + if self.stopped_early { + return ScanStep::Resume(ScanProgress { + resume_after: self.last_checked.clone().unwrap_or(started.resume_after), + }); + } + let covered_every_account = started.resume_after.is_empty(); + if covered_every_account && self.retryable == 0 && self.failed == 0 { + ScanStep::Completed { + unresolved: self.unresolved, + } + } else { + ScanStep::Resume(ScanProgress::default()) + } + } +} + +pub(crate) async fn run_once( + state: &AppState, + store: &RecoveryStore, + network_type: MidenNetworkType, +) -> Result<(), StartupRecoveryError> { + run_within(state, store, network_type, SCAN_BUDGET, STALE_LEASE_WAIT).await +} + +/// Takes the recovery lease, waiting out one a stopped process left behind. A lease that is +/// still held after `wait` belongs to a live process. +async fn acquire_lease( + store: &RecoveryStore, + wait: Duration, +) -> Result { + let give_up_at = Instant::now() + wait; + let mut waiting = false; + loop { + let acquired = bounded_db_call( + "acquire the recovery lease", + store.leader.try_acquire(LEASE_TTL), + ) + .await + .map_err(StartupRecoveryError::LeaseUnavailable)?; + match acquired { + Some(lease) => return Ok(lease), + None if Instant::now() + LEASE_RETRY_INTERVAL > give_up_at => { + return Err(StartupRecoveryError::LeaseBusy); + } + None => { + if !waiting { + tracing::info!("Waiting for the account recovery lease another process holds"); + waiting = true; + } + tokio::time::sleep(LEASE_RETRY_INTERVAL).await + } + } + } +} + +async fn run_within( + state: &AppState, + store: &RecoveryStore, + network_type: MidenNetworkType, + budget: Duration, + stale_lease_wait: Duration, +) -> Result<(), StartupRecoveryError> { + let lease = acquire_lease(store, stale_lease_wait).await?; + let cancel = CancellationToken::new(); + let renewal = spawn_lease_renewal( + store.leader.clone(), + lease.clone(), + LEASE_TTL, + LEASE_TTL / 3, + cancel.clone(), + "Account recovery", + ); + let result = async { + if bounded_db_call( + "read the recovery checkpoint", + store.startup_completed(network_type), + ) + .await? + { + tracing::info!( + ?network_type, + "Account recovery already completed; skipping the scan" + ); + return Ok(()); + } + let started = + bounded_db_call("read recovery progress", store.progress(network_type)).await?; + let summary = run( + state, + store, + network_type, + &lease, + &cancel, + &started.resume_after, + Instant::now() + budget, + ) + .await?; + match summary.next_step(started) { + ScanStep::Completed { unresolved } => { + bounded_db_call( + "record recovery completion", + store.complete_startup(network_type, unresolved, &lease), + ) + .await?; + tracing::info!( + unresolved, + "Account recovery completed; later startups skip the scan" + ); + Ok(()) + } + ScanStep::Resume(progress) => { + tracing::warn!( + resume_after = %progress.resume_after, + "Account recovery will continue on the next eligible startup" + ); + bounded_db_call( + "save recovery progress", + store.save_progress(network_type, &progress, &lease), + ) + .await + } + } + } + .await; + let held = bounded_db_call( + "confirm the recovery lease", + store.leader.verify_held(&lease), + ) + .await; + renewal.abort(); + let _ = renewal.await; + let released = bounded_db_call("release the recovery lease", store.leader.release(lease)).await; + if !held.map_err(StartupRecoveryError::LeaseUnavailable)? { + return Err(StartupRecoveryError::LeaseUnavailable( + "The recovery lease was lost".into(), + )); + } + result + .and(released) + .map_err(StartupRecoveryError::Incomplete) +} + +async fn bounded_db_call( + operation: &str, + work: impl Future>, +) -> Result { + timeout(SCAN_CONTROL_BUDGET, work) + .await + .map_err(|_| format!("Timed out while trying to {operation}"))? + .map_err(|error| error.to_string()) +} + +async fn run( + state: &AppState, + store: &RecoveryStore, + network_type: MidenNetworkType, + lease: &Lease, + cancel: &CancellationToken, + resume_after: &str, + deadline: Instant, +) -> Result { + tracing::info!(?network_type, %resume_after, "Starting account recovery scan"); + let mut after = resume_after.to_owned(); + let mut summary = ScanSummary::default(); + let mut consecutive_failed_checks = 0; + let mut position_before_failed_checks = None; + 'scan: loop { + let candidates = + match timeout_at(deadline, store.candidates(network_type, &after, PAGE_SIZE)).await { + Ok(result) => result?, + Err(_) => { + summary.stopped_early = true; + break; + } + }; + let Some(last) = candidates.last() else { break }; + after.clone_from(&last.account_id); + + for candidate in candidates { + if cancel.is_cancelled() { + return Err("account recovery lease lost".into()); + } + let result = match timeout_at(deadline, check_candidate(state, &candidate)).await { + Ok(result) => result, + Err(_) => { + summary.stopped_early = true; + break 'scan; + } + }; + if cancel.is_cancelled() { + return Err("account recovery lease lost".into()); + } + summary.checked += 1; + consecutive_failed_checks = match &result { + Ok(_) => 0, + Err(_) => consecutive_failed_checks + 1, + }; + let record = match result { + Ok(RecoveryOutcome::Unchanged) => { + summary.unchanged += 1; + PendingRecord::Clear + } + Ok(RecoveryOutcome::Verified(applied, auth)) => { + match store + .apply( + &candidate, + &applied.state_json, + &auth, + state.clock.now(), + lease, + ) + .await + { + Ok(true) => { + summary.restored += 1; + tracing::info!(account_id = %candidate.account_id, "Restored chain-verified account state and signers"); + PendingRecord::Clear + } + Ok(false) => { + summary.retryable += 1; + tracing::info!(account_id = %candidate.account_id, "Account became ineligible or changed during recovery"); + PendingRecord::Write( + PendingOutcome::Deferred, + "the account changed or became ineligible during recovery".into(), + ) + } + Err(reason) => { + summary.failed += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery transaction failed"); + PendingRecord::Write( + PendingOutcome::Deferred, + "restore transaction failed; see Guardian logs".into(), + ) + } + } + } + Ok(RecoveryOutcome::Retryable(reason)) => { + summary.retryable += 1; + tracing::info!(account_id = %candidate.account_id, %reason, "Account recovery deferred until a later startup"); + PendingRecord::Write(PendingOutcome::Deferred, reason) + } + Ok(RecoveryOutcome::Unresolved(reason)) => { + summary.unresolved += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery needs additional evidence"); + PendingRecord::Write(PendingOutcome::Unresolved, reason) + } + Err(reason) => { + summary.failed += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery could not check the candidate"); + PendingRecord::Write( + PendingOutcome::Deferred, + "account verification failed; see Guardian logs".into(), + ) + } + }; + let recorded = match record { + PendingRecord::Clear => { + bounded_db_call( + "clear a pending recovery record", + store.clear_pending(&candidate.account_id, lease), + ) + .await + } + PendingRecord::Write(outcome, reason) => { + bounded_db_call( + "write a pending recovery record", + store.record_pending(&candidate, outcome, &reason, lease), + ) + .await + } + }; + if let Err(reason) = recorded { + summary.failed += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery could not update the pending record"); + } + summary.last_checked = Some(candidate.account_id); + if consecutive_failed_checks == 0 { + position_before_failed_checks.clone_from(&summary.last_checked); + } + if consecutive_failed_checks >= MAX_CONSECUTIVE_FAILED_CHECKS { + tracing::warn!( + consecutive_failed_checks, + "Account recovery stopped early after repeated failed checks" + ); + summary.last_checked = position_before_failed_checks; + summary.stopped_early = true; + break 'scan; + } + if Instant::now() >= deadline { + summary.stopped_early = true; + break 'scan; + } + } + } + let remaining = if summary.stopped_early { + let position = summary.last_checked.as_deref().unwrap_or(resume_after); + timeout( + REMAINING_COUNT_BUDGET, + store.remaining_candidates(network_type, position), + ) + .await + .ok() + .and_then(Result::ok) + } else { + Some(0) + }; + tracing::info!( + checked = summary.checked, + restored = summary.restored, + unchanged = summary.unchanged, + retryable = summary.retryable, + unresolved = summary.unresolved, + failed = summary.failed, + stopped_early = summary.stopped_early, + ?remaining, + "Finished account recovery scan" + ); + Ok(summary) +} + +async fn check_candidate( + state: &AppState, + candidate: &RecoveryCandidate, +) -> Result { + let chain = state + .network_client + .verify_commitment( + &candidate.account_id, + &candidate.stored_commitment, + RpcReadMode::Configured, + ) + .await?; + if matches!(chain, StateVerification::Match) { + return Ok(RecoveryOutcome::Unchanged); + } + if matches!(chain, StateVerification::Absent) { + if candidate.canonical_nonce.is_some() { + return Ok(RecoveryOutcome::Retryable( + "the configured node has no record of the account".into(), + )); + } + if candidate.legacy_network_label || candidate.stored_nonce == Some(0) { + return Ok(RecoveryOutcome::Unchanged); + } + } + if let Some(unsettled_work) = &candidate.unsettled_work { + return Ok(RecoveryOutcome::Retryable(unsettled_work.clone())); + } + + let (Some(canonical_nonce), Some(target)) = ( + candidate.canonical_nonce, + candidate.canonical_commitment.as_deref(), + ) else { + return Ok(RecoveryOutcome::Unresolved( + "no canonical history supplies a recovery target".into(), + )); + }; + if !matches!(&chain, StateVerification::Mismatch { on_chain } if on_chain == target) { + return Ok(RecoveryOutcome::Retryable( + "latest canonical delta does not match the current chain head".into(), + )); + } + let Some(count) = u32::try_from(canonical_nonce) + .ok() + .filter(|count| (1..=MAX_DELTAS).contains(count)) + else { + return Ok(RecoveryOutcome::Unresolved( + "canonical history exceeds the recovery limit".into(), + )); + }; + let history = state + .storage + .list_canonical_deltas_paged(&candidate.account_id, count, None) + .await; + let deltas = match history { + Ok(deltas) => deltas, + Err(reason) if is_record_corruption(&reason) => { + return Ok(RecoveryOutcome::Unresolved(reason)); + } + Err(reason) => return Err(reason), + }; + if deltas.len() != count as usize { + return Ok(RecoveryOutcome::Unresolved( + "missing canonical deltas from account creation".into(), + )); + } + + let stored = if candidate.stored_nonce.is_none() { + match state.storage.pull_state(&candidate.account_id).await { + Ok(stored) if stored.commitment == candidate.stored_commitment => Some(stored), + Ok(_) => { + return Ok(RecoveryOutcome::Retryable( + "stored commitment changed during recovery".into(), + )); + } + Err(reason) if is_record_corruption(&reason) => { + return Ok(RecoveryOutcome::Unresolved(reason)); + } + Err(reason) => return Err(reason), + } + } else { + None + }; + let stored_nonce = candidate + .stored_nonce + .and_then(|nonce| u64::try_from(nonce).ok()); + let network = state.network_client.clone(); + let account_id = candidate.account_id.clone(); + let target = target.to_owned(); + let proof = reconstructor() + .run_background(move || { + prove_recovery( + network.as_ref(), + &account_id, + &target, + deltas, + stored_nonce, + stored, + ) + }) + .await; + let (applied, auth) = match proof { + Ok(Some(proof)) => proof, + Ok(None) => { + return Ok(RecoveryOutcome::Unresolved( + "the stored state is newer than the verified history".into(), + )); + } + Err(ReconstructError::Operation(reason)) => return Ok(RecoveryOutcome::Unresolved(reason)), + Err(ReconstructError::Task(reason)) => { + return Err(format!("state reconstruction failed: {reason}")); + } + }; + + let scheme = auth.guardian_scheme(); + if applied.guardian_commitment.as_deref() != Some(state.ack.commitment(&scheme).as_str()) { + return Ok(RecoveryOutcome::Unresolved( + "recovered state is bound to a different Guardian key".into(), + )); + } + + match state + .network_client + .verify_commitment( + &candidate.account_id, + &applied.commitment, + RpcReadMode::Configured, + ) + .await? + { + StateVerification::Match => {} + StateVerification::Mismatch { .. } | StateVerification::Absent => { + return Ok(RecoveryOutcome::Retryable( + "reconstructed state does not match the current chain head".into(), + )); + } + } + + Ok(RecoveryOutcome::Verified(Box::new(applied), auth)) +} + +fn prove_recovery( + network: &dyn NetworkClient, + account_id: &str, + target: &str, + deltas: Vec, + stored_nonce: Option, + stored: Option, +) -> Result, String> { + let applied = reconstruct(network, account_id, target, deltas)?; + let nonce = match stored { + Some(stored) => { + let head = network.get_state_head(account_id, &stored.state_json)?; + if head.commitment != stored.commitment { + return Err("stored state does not match its recorded commitment".into()); + } + head.nonce + } + None => stored_nonce, + } + .ok_or("stored state has no verifiable nonce")?; + let recovered_nonce = applied.nonce.ok_or("recovered state has no nonce")?; + if nonce > recovered_nonce { + return Ok(None); + } + + let binding = network + .account_auth_binding(&applied.state_json)? + .ok_or("missing account auth binding")?; + network.validate_multisig_auth(&applied.state_json)?; + let scheme = binding + .guardian_scheme + .ok_or("missing on-chain Guardian signature scheme")?; + let auth = recovered_auth(scheme, &applied)?; + Ok(Some((applied, auth))) +} + +fn reconstruct( + network: &dyn NetworkClient, + account_id: &str, + target: &str, + deltas: Vec, +) -> Result { + let mut reconstructed: Option = None; + for (index, delta) in deltas.into_iter().rev().enumerate() { + if delta.account_id != account_id + || delta.nonce != index as u64 + 1 + || !delta.status.is_canonical() + { + return Err("canonical history is not contiguous from account creation".into()); + } + let applied = match reconstructed { + None => network.apply_delta(&serde_json::Value::Null, &delta.delta_payload)?, + Some(previous) => network.verify_and_apply_delta( + &delta.prev_commitment, + &previous.state_json, + &delta.delta_payload, + )?, + }; + if delta.new_commitment.as_deref() != Some(applied.commitment.as_str()) + || applied.nonce != Some(delta.nonce) + { + return Err("reconstructed delta does not match its canonical row".into()); + } + reconstructed = Some(applied); + } + let applied = reconstructed.ok_or("canonical history is empty")?; + let head = network.get_state_head(account_id, &applied.state_json)?; + if head.commitment != target || head.nonce != applied.nonce { + return Err("reconstructed state does not match the expected account head".into()); + } + Ok(applied) +} + +fn recovered_auth(scheme: SignatureScheme, state: &AppliedState) -> Result { + if state.cosigners.is_empty() || state.cosigners.iter().any(|signer| signer.scheme.is_none()) { + return Err("recovered signer set is empty or has an unknown signature scheme".into()); + } + let baseline = match scheme { + SignatureScheme::Falcon => Auth::MidenFalconRpo { + cosigner_commitments: Vec::new(), + }, + SignatureScheme::Ecdsa => Auth::MidenEcdsa { + cosigner_commitments: Vec::new(), + }, + }; + Ok(baseline.with_chain_cosigners(state.cosigners.clone())) +} + +#[cfg(test)] +mod postgres_tests; diff --git a/crates/server/src/jobs/account_recovery/postgres_tests.rs b/crates/server/src/jobs/account_recovery/postgres_tests.rs new file mode 100644 index 00000000..39ea363c --- /dev/null +++ b/crates/server/src/jobs/account_recovery/postgres_tests.rs @@ -0,0 +1,1080 @@ +use std::sync::Arc; + +use base64::Engine; +use diesel::sql_types::{BigInt, Jsonb, Text}; +use diesel_async::RunQueryDsl; +use guardian_shared::{ + FromJson, ToJson, + auth_request_payload::AuthRequestPayload, + hex::{FromHex, IntoHex}, +}; +use miden_protocol::account::{ + Account, AccountCodePatch, AccountDelta, AccountStoragePatch, AccountVaultDelta, +}; +use miden_protocol::block::BlockNumber; +use miden_protocol::transaction::{ + InputNotes, RawOutputNotes, TransactionSummary, TransactionSummaryUserParams, +}; +use miden_protocol::{Felt, Word}; + +use crate::delta_object::DeltaStatus; +use crate::metadata::{ + AccountMetadata, MetadataStore, NetworkConfig, postgres::PostgresMetadataStore, +}; +use crate::network::{NetworkType, miden::MidenNetworkClient}; +use crate::services::{GetStateParams, get_state}; +use crate::state_object::StateObject; +use crate::storage::encryption::{ + cipher::{Aes256GcmCipher, StorageCipher}, + decorator::EncryptedStorage, + envelope::RecordAad, + key_provider::InMemoryKeyProvider, +}; +use crate::storage::{StorageBackend, postgres::PostgresService}; +use crate::testing::helpers::{ + IntegrationMockNetworkClient, TestEcdsaSigner, build_guarded_multisig_account, + create_test_app_state_with_mocks, +}; + +use super::*; + +fn account_at_nonce(account: &Account, nonce: Felt) -> Account { + Account::new_existing( + account.id(), + account.vault().clone(), + account.storage().clone(), + account.code().clone(), + nonce, + ) +} + +fn canonical_delta( + network: &dyn NetworkClient, + delta: AccountDelta, + prev_commitment: String, + base_state: &serde_json::Value, +) -> (DeltaObject, AppliedState) { + let account_id = delta.id().to_hex(); + let payload = TransactionSummary::new( + delta, + InputNotes::new(Vec::new()).unwrap(), + RawOutputNotes::new(Vec::new()).unwrap(), + BlockNumber::from(0), + Word::default(), + 0, + TransactionSummaryUserParams::new([Felt::ZERO; 6]), + ) + .to_json(); + let applied = network.apply_delta(base_state, &payload).unwrap(); + let row = DeltaObject { + account_id, + nonce: applied.nonce.unwrap(), + prev_commitment, + new_commitment: Some(applied.commitment.clone()), + delta_payload: payload, + status: DeltaStatus::canonical("2026-11-14T12:00:00Z".into()), + ..Default::default() + }; + (row, applied) +} + +fn canonical_history( + account: &Account, + network: &dyn NetworkClient, +) -> (Vec, AppliedState) { + let (first, first_state) = canonical_delta( + network, + AccountDelta::try_from(account_at_nonce(account, Felt::ONE)).unwrap(), + account.to_commitment().into_hex(), + &serde_json::Value::Null, + ); + let (second, second_state) = canonical_delta( + network, + AccountDelta::new( + account.id(), + AccountStoragePatch::default(), + AccountVaultDelta::default(), + AccountCodePatch::default(), + Felt::ONE, + ) + .unwrap(), + first_state.commitment, + &first_state.state_json, + ); + (vec![second, first], second_state) +} + +#[tokio::test] +async fn rejects_incomplete_history_and_commitment_mismatches() { + let network = MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal); + let scheme = SignatureScheme::Ecdsa; + let account = build_guarded_multisig_account( + [25; 32], + 1, + &[(Word::from([1, 2, 3, 4_u32]), scheme)], + (Word::from([5, 6, 7, 8_u32]), scheme), + ); + let (mut deltas, expected) = canonical_history(&account, &network); + let account_id = account.id().to_hex(); + let target = &expected.commitment; + assert_eq!( + reconstruct(&network, &account_id, target, deltas.clone()).unwrap(), + expected + ); + assert!( + prove_recovery(&network, &account_id, target, deltas.clone(), Some(1), None) + .unwrap() + .is_some() + ); + assert!( + prove_recovery(&network, &account_id, target, deltas.clone(), Some(3), None) + .unwrap() + .is_none() + ); + assert!(reconstruct(&network, &account_id, target, vec![deltas[0].clone()]).is_err()); + deltas[1].new_commitment = Some("wrong commitment".into()); + assert!(reconstruct(&network, &account_id, target, deltas).is_err()); +} + +#[tokio::test] +async fn partial_history_fixture_cannot_rebuild_a_missing_base() { + let network = MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal); + let delta: DeltaObject = serde_json::from_str(crate::testing::fixtures::DELTA_1_JSON).unwrap(); + let summary = TransactionSummary::from_json(&delta.delta_payload).unwrap(); + assert!(summary.account_delta().code().is_empty()); + assert!( + network + .apply_delta(&serde_json::Value::Null, &delta.delta_payload) + .is_err() + ); +} + +#[tokio::test] +#[ignore = "requires a dedicated Postgres test database"] +async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_evidence() { + let url = crate::testing::pg::test_database_url().await; + let raw = Arc::new(PostgresService::new(&url, 4).await.unwrap()); + let metadata = Arc::new(PostgresMetadataStore::new(&url, 4).await.unwrap()); + let key = base64::engine::general_purpose::STANDARD.encode([2_u8; 32]); + let provider = InMemoryKeyProvider::from_dev_key(&key, "recovery-test").unwrap(); + let cipher = Arc::new(Aes256GcmCipher::new(Arc::new(provider))); + let storage = Arc::new(EncryptedStorage::new(raw, cipher.clone())); + let store = RecoveryStore::new( + metadata.pool_handle(), + Some(cipher.clone()), + "test-recovery", + ); + let contender = RecoveryStore::new( + metadata.pool_handle(), + Some(cipher.clone()), + "other-recovery", + ); + let lease = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + let chain = Arc::new(IntegrationMockNetworkClient::new( + MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal), + )); + let state = create_test_app_state_with_mocks(storage.clone(), chain.clone(), metadata.clone()); + let scheme = SignatureScheme::Ecdsa; + let signer = TestEcdsaSigner::new(); + let account = build_guarded_multisig_account( + [25; 32], + 1, + &[(Word::from_hex(&signer.commitment_hex).unwrap(), scheme)], + ( + Word::from_hex(&state.ack.commitment(&scheme)).unwrap(), + scheme, + ), + ); + let (deltas, canonical) = canonical_history(&account, chain.as_ref()); + let account_id = &deltas[0].account_id; + let target = canonical.commitment.as_str(); + let old_auth = Auth::MidenEcdsa { + cosigner_commitments: vec![Word::from([8, 7, 6, 5_u32]).into_hex()], + }; + let expected_auth = Auth::MidenEcdsa { + cosigner_commitments: vec![signer.commitment_hex.clone()], + }; + let other_state = account_at_nonce(&account, Felt::from(2_u8)); + let prior = StateObject { + account_id: account_id.clone(), + state_json: other_state.to_json(), + commitment: other_state.to_commitment().into_hex(), + nonce: Some(2), + created_at: "2026-11-14T12:00:00Z".into(), + updated_at: "2026-11-14T13:00:00Z".into(), + auth_scheme: "ecdsa".into(), + }; + metadata + .set(AccountMetadata { + account_id: account_id.clone(), + auth: old_auth.clone(), + network_config: NetworkConfig::miden_default(), + created_at: prior.created_at.clone(), + updated_at: prior.updated_at.clone(), + has_pending_candidate: false, + paused_at: None, + paused_reason: None, + released_at: None, + }) + .await + .unwrap(); + storage + .submit_state(&StateObject { + state_json: canonical.state_json.clone(), + commitment: canonical.commitment.clone(), + updated_at: prior.created_at.clone(), + ..prior.clone() + }) + .await + .unwrap(); + storage.replace_state(&prior, target).await.unwrap(); + for delta in &deltas { + storage.submit_delta(delta).await.unwrap(); + } + let testnet = NetworkConfig::Miden { + network_type: MidenNetworkType::Testnet, + }; + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("INSERT INTO account_recovery_startup (network_config) VALUES ($1)") + .bind::(serde_json::to_value(&testnet).unwrap()) + .execute(&mut conn) + .await + .unwrap(); + for label in [&testnet, &NetworkConfig::miden_default()] { + diesel::sql_query( + "UPDATE account_metadata SET network_config = $2 WHERE account_id = $1", + ) + .bind::(account_id) + .bind::(serde_json::to_value(label).unwrap()) + .execute(&mut conn) + .await + .unwrap(); + let candidates = store + .candidates(MidenNetworkType::Mainnet, "", 10) + .await + .unwrap(); + assert_eq!(candidates.len(), usize::from(label != &testnet)); + } + } + assert_eq!( + storage.pull_state_head(account_id).await.unwrap().nonce, + None + ); + assert_eq!( + metadata + .get_for_read(account_id) + .await + .unwrap() + .unwrap() + .auth, + old_auth + ); + let mut candidates = store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap(); + assert_eq!(candidates.len(), 1); + let candidate = candidates.pop().unwrap(); + assert_eq!(candidate.canonical_nonce, Some(2)); + assert_eq!(candidate.stored_nonce, None); + assert!(candidate.legacy_network_label); + + let mut absent = state.clone(); + absent.network_client = Arc::new( + crate::testing::mocks::MockNetworkClient::new() + .with_verify_commitment(Ok(StateVerification::Absent)) + .with_verify_commitment(Ok(StateVerification::Absent)), + ); + assert!(matches!( + check_candidate(&absent, &candidate).await.unwrap(), + RecoveryOutcome::Retryable(reason) if reason.contains("no record of the account") + )); + let mut undeployed = store + .candidates(MidenNetworkType::Local, "", 10) + .await + .unwrap() + .pop() + .unwrap(); + assert!(!undeployed.legacy_network_label); + undeployed.stored_nonce = Some(0); + undeployed.canonical_nonce = None; + undeployed.canonical_commitment = None; + assert!(matches!( + check_candidate(&absent, &undeployed).await.unwrap(), + RecoveryOutcome::Unchanged + )); + let mut labelled_without_history = store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap() + .pop() + .unwrap(); + assert!(labelled_without_history.legacy_network_label); + labelled_without_history.stored_nonce = Some(5); + labelled_without_history.canonical_nonce = None; + labelled_without_history.canonical_commitment = None; + let mut still_absent = state.clone(); + still_absent.network_client = Arc::new( + crate::testing::mocks::MockNetworkClient::new() + .with_verify_commitment(Ok(StateVerification::Absent)), + ); + assert!(matches!( + check_candidate(&still_absent, &labelled_without_history) + .await + .unwrap(), + RecoveryOutcome::Unchanged + )); + + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("BEGIN").execute(&mut conn).await.unwrap(); + diesel::sql_query("LOCK TABLE states IN ACCESS EXCLUSIVE MODE") + .execute(&mut conn) + .await + .unwrap(); + let timed_out = run( + &state, + &store, + MidenNetworkType::Testnet, + &lease, + &CancellationToken::new(), + "", + Instant::now() + Duration::from_millis(40), + ) + .await + .unwrap(); + diesel::sql_query("ROLLBACK") + .execute(&mut conn) + .await + .unwrap(); + assert!(timed_out.stopped_early); + assert_eq!(timed_out.checked, 0); + } + + chain.set_on_chain_commitment(account_id, "different head"); + assert!(matches!( + check_candidate(&state, &candidate).await.unwrap(), + RecoveryOutcome::Retryable(reason) if reason.contains("current chain head") + )); + chain.set_on_chain_commitment(account_id, target); + let mut ahead = store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap() + .pop() + .unwrap(); + ahead.stored_nonce = Some(9); + assert!(matches!( + check_candidate(&state, &ahead).await.unwrap(), + RecoveryOutcome::Unresolved(reason) if reason.contains("newer than the verified history") + )); + let pending = DeltaObject { + nonce: 3, + prev_commitment: target.into(), + new_commitment: None, + status: DeltaStatus::candidate("2026-11-14T12:30:00Z".into()), + ..deltas[0].clone() + }; + storage.submit_delta(&pending).await.unwrap(); + assert!(matches!( + check_candidate(&state, &candidate).await.unwrap(), + RecoveryOutcome::Verified(_, _) + )); + assert!( + !store + .apply( + &candidate, + &canonical.state_json, + &expected_auth, + state.clock.now(), + &lease + ) + .await + .unwrap() + ); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment + ); + assert!(matches!( + run_within( + &state, + &contender, + MidenNetworkType::Testnet, + SCAN_BUDGET, + Duration::ZERO + ) + .await, + Err(StartupRecoveryError::LeaseBusy) + )); + store.leader.release(lease.clone()).await.unwrap(); + + run_once(&state, &store, MidenNetworkType::Testnet) + .await + .unwrap(); + assert!( + !store + .startup_completed(MidenNetworkType::Testnet) + .await + .unwrap() + ); + + storage + .update_delta_status( + account_id, + 3, + DeltaStatus::Discarded { + timestamp: "2026-11-14T12:40:00Z".into(), + reason: None, + }, + ) + .await + .unwrap(); + + assert!( + store + .complete_startup(MidenNetworkType::Testnet, 0, &lease) + .await + .is_err() + ); + + let mut unavailable = state.clone(); + unavailable.network_client = Arc::new( + crate::testing::mocks::MockNetworkClient::new() + .with_verify_commitment(Err("simulated RPC outage".into())), + ); + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("DELETE FROM account_recovery_pending WHERE account_id = $1") + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + run_once(&unavailable, &store, MidenNetworkType::Testnet) + .await + .unwrap(); + assert!( + !store + .startup_completed(MidenNetworkType::Testnet) + .await + .unwrap() + ); + + #[derive(diesel::QueryableByName)] + struct DeferredReason { + #[diesel(sql_type = Text)] + reason: String, + } + let deferred = diesel::sql_query( + "SELECT reason FROM account_recovery_pending WHERE account_id = $1 AND outcome = 'deferred'", + ) + .bind::(account_id) + .get_result::(&mut conn) + .await + .unwrap(); + assert!(deferred.reason.contains("account verification failed")); + + diesel::sql_query("BEGIN").execute(&mut conn).await.unwrap(); + diesel::sql_query("LOCK TABLE account_recovery_backups IN ACCESS EXCLUSIVE MODE") + .execute(&mut conn) + .await + .unwrap(); + let blocked_restore = timeout( + Duration::from_secs(8), + run_once(&state, &store, MidenNetworkType::Testnet), + ) + .await; + diesel::sql_query("ROLLBACK") + .execute(&mut conn) + .await + .unwrap(); + blocked_restore.unwrap().unwrap(); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment + ); + let deferred = diesel::sql_query( + "SELECT reason FROM account_recovery_pending WHERE account_id = $1 AND outcome = 'deferred'", + ) + .bind::(account_id) + .get_result::(&mut conn) + .await + .unwrap(); + assert!(deferred.reason.contains("restore transaction failed")); + drop(conn); + + run_once(&state, &store, MidenNetworkType::Testnet) + .await + .unwrap(); + assert!( + store + .startup_completed(MidenNetworkType::Testnet) + .await + .unwrap() + ); + assert!( + !store + .startup_completed(MidenNetworkType::Local) + .await + .unwrap() + ); + assert_eq!( + storage.pull_state_head(account_id).await.unwrap().nonce, + Some(2) + ); + + let held = contender + .leader + .try_acquire(LEASE_TTL) + .await + .unwrap() + .unwrap(); + assert!(matches!( + run_within( + &state, + &store, + MidenNetworkType::Testnet, + SCAN_BUDGET, + Duration::ZERO + ) + .await, + Err(StartupRecoveryError::LeaseBusy) + )); + assert!(contender.leader.verify_held(&held).await.unwrap()); + contender.leader.release(held).await.unwrap(); + run_once(&state, &store, MidenNetworkType::Testnet) + .await + .unwrap(); + assert!( + !store + .apply( + &candidate, + &canonical.state_json, + &expected_auth, + state.clock.now(), + &lease + ) + .await + .unwrap() + ); + let payload = + AuthRequestPayload::from_json_serializable(&serde_json::json!({"account_id": account_id})) + .unwrap(); + let (signature, timestamp) = signer.sign_request(account_id, &payload); + let restored = get_state( + &state, + GetStateParams { + account_id: account_id.clone(), + credentials: crate::metadata::Credentials::signature( + signer.pubkey_hex, + signature, + timestamp, + ) + .with_request_payload(payload), + }, + ) + .await + .unwrap() + .state; + assert_eq!( + chain + .get_state_head(account_id, &restored.state_json) + .unwrap() + .commitment, + target + ); + assert_eq!( + metadata + .get_for_read(account_id) + .await + .unwrap() + .unwrap() + .auth, + expected_auth + ); + + #[derive(diesel::QueryableByName)] + struct Backup { + #[diesel(sql_type = BigInt)] + total: i64, + #[diesel(sql_type = Jsonb)] + prior_state: serde_json::Value, + #[diesel(sql_type = Jsonb)] + prior_auth: serde_json::Value, + } + let mut conn = metadata.pool_handle().get().await.unwrap(); + let backup = diesel::sql_query("SELECT count(*) OVER () AS total, prior_state, prior_auth FROM account_recovery_backups WHERE account_id = $1") + .bind::(account_id).get_result::(&mut conn).await.unwrap(); + let old_payload = &backup.prior_state["state_json"]; + assert_ne!(old_payload, &prior.state_json); + assert_eq!( + cipher + .decrypt(&RecordAad::State { account_id }, old_payload) + .unwrap(), + prior.state_json + ); + assert_eq!( + serde_json::from_value::(backup.prior_auth).unwrap(), + old_auth + ); + assert_eq!(backup.total, 1); + + #[derive(diesel::QueryableByName)] + struct PendingCount { + #[diesel(sql_type = BigInt)] + pending: i64, + } + let pending = diesel::sql_query( + "SELECT count(*) AS pending FROM account_recovery_pending WHERE account_id = $1", + ) + .bind::(account_id) + .get_result::(&mut conn) + .await + .unwrap(); + assert_eq!(pending.pending, 0); +} + +fn scan_progress(resume_after: &str) -> ScanProgress { + ScanProgress { + resume_after: resume_after.into(), + } +} + +#[test] +fn a_scan_completes_only_from_a_whole_pass_that_deferred_no_account() { + let interrupted = ScanSummary { + unresolved: 2, + last_checked: Some("0xb".into()), + stopped_early: true, + ..Default::default() + }; + assert_eq!( + interrupted.next_step(scan_progress("0xa")), + ScanStep::Resume(scan_progress("0xb")) + ); + + let stalled = ScanSummary { + stopped_early: true, + ..Default::default() + }; + assert_eq!( + stalled.next_step(scan_progress("0xa")), + ScanStep::Resume(scan_progress("0xa")) + ); + + let finished = ScanSummary { + unresolved: 1, + last_checked: Some("0xc".into()), + ..Default::default() + }; + assert_eq!( + finished.next_step(ScanProgress::default()), + ScanStep::Completed { unresolved: 1 } + ); + assert_eq!( + finished.next_step(scan_progress("0xa")), + ScanStep::Resume(ScanProgress::default()) + ); + + for deferring in [ + ScanSummary { + retryable: 1, + ..Default::default() + }, + ScanSummary { + failed: 1, + ..Default::default() + }, + ] { + assert_eq!( + deferring.next_step(ScanProgress::default()), + ScanStep::Resume(ScanProgress::default()) + ); + } +} + +#[tokio::test] +#[ignore = "requires a dedicated Postgres test database"] +async fn account_recovery_scan_continues_from_its_saved_position_and_records_pending_accounts() { + let url = crate::testing::pg::test_database_url().await; + let storage = Arc::new(PostgresService::new(&url, 4).await.unwrap()); + let metadata = Arc::new(PostgresMetadataStore::new(&url, 4).await.unwrap()); + let store = RecoveryStore::new(metadata.pool_handle(), None, "resume-recovery"); + let network = MidenNetworkType::Cedar; + let first = "0xresume0000000000000000000000a"; + let second = "0xresume0000000000000000000000b"; + let joined_later = "0xresume0000000000000000000000c"; + for account_id in [first, second, joined_later] { + metadata + .set(AccountMetadata { + account_id: account_id.into(), + auth: Auth::MidenEcdsa { + cosigner_commitments: vec![Word::from([1, 2, 3, 4_u32]).into_hex()], + }, + network_config: NetworkConfig::Miden { + network_type: network, + }, + created_at: "2026-11-14T12:00:00Z".into(), + updated_at: "2026-11-14T12:00:00Z".into(), + has_pending_candidate: false, + paused_at: None, + paused_reason: None, + released_at: None, + }) + .await + .unwrap(); + storage + .submit_state(&StateObject { + account_id: account_id.into(), + state_json: serde_json::json!({}), + commitment: format!("{account_id}-stored"), + nonce: Some(1), + created_at: "2026-11-14T12:00:00Z".into(), + updated_at: "2026-11-14T13:00:00Z".into(), + auth_scheme: "ecdsa".into(), + }) + .await + .unwrap(); + } + for (nonce, prev_commitment, new_commitment) in [ + (3, "0xheld-base", "0xheld-middle"), + (4, "0xheld-middle", "0xheld-head"), + ] { + storage + .submit_delta(&DeltaObject { + account_id: joined_later.into(), + nonce, + prev_commitment: prev_commitment.into(), + new_commitment: Some(new_commitment.into()), + delta_payload: serde_json::json!({}), + status: DeltaStatus::canonical("2026-11-14T12:30:00Z".into()), + ..Default::default() + }) + .await + .unwrap(); + } + let chain_head = |account_id: &str| StateVerification::Mismatch { + on_chain: if account_id == joined_later { + "0xheld-head".into() + } else { + "0xelsewhere".into() + }, + }; + let checking = |accounts: &[&str]| { + let chain = accounts.iter().rev().fold( + crate::testing::mocks::MockNetworkClient::new(), + |chain, account_id| chain.with_verify_commitment(Ok(chain_head(account_id))), + ); + create_test_app_state_with_mocks(storage.clone(), Arc::new(chain), metadata.clone()) + }; + let lease = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + let later = || Instant::now() + Duration::from_secs(60); + + let whole = run( + &checking(&[first, second, joined_later]), + &store, + network, + &lease, + &CancellationToken::new(), + "", + later(), + ) + .await + .unwrap(); + assert_eq!((whole.checked, whole.unresolved), (3, 3)); + assert_eq!(whole.last_checked.as_deref(), Some(joined_later)); + assert!(!whole.stopped_early); + + #[derive(diesel::QueryableByName, Debug, PartialEq, Eq)] + struct Pending { + #[diesel(sql_type = Text)] + account_id: String, + #[diesel(sql_type = Text)] + outcome: String, + #[diesel(sql_type = Text)] + reason: String, + #[diesel(sql_type = diesel::sql_types::Nullable)] + canonical_nonce: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + history_starts_at_nonce: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + history_base_commitment: Option, + } + let pending_accounts = || async { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "SELECT account_id, outcome, reason, canonical_nonce, history_starts_at_nonce, + history_base_commitment + FROM account_recovery_pending WHERE account_id LIKE '0xresume%' ORDER BY account_id", + ) + .load::(&mut conn) + .await + .unwrap() + }; + let without_history = |account_id: &str| Pending { + account_id: account_id.into(), + outcome: "unresolved".into(), + reason: "no canonical history supplies a recovery target".into(), + canonical_nonce: None, + history_starts_at_nonce: None, + history_base_commitment: None, + }; + assert_eq!( + pending_accounts().await, + [ + without_history(first), + without_history(second), + Pending { + account_id: joined_later.into(), + outcome: "unresolved".into(), + reason: "missing canonical deltas from account creation".into(), + canonical_nonce: Some(4), + history_starts_at_nonce: Some(3), + history_base_commitment: Some("0xheld-base".into()), + }, + ] + ); + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("BEGIN").execute(&mut conn).await.unwrap(); + diesel::sql_query("LOCK TABLE account_recovery_pending, account_recovery_startup IN ACCESS EXCLUSIVE MODE") + .execute(&mut conn) + .await + .unwrap(); + let clear = timeout(Duration::from_secs(5), store.clear_pending(second, &lease)).await; + let progress = ScanProgress::default(); + let save = timeout( + Duration::from_secs(5), + store.save_progress(network, &progress, &lease), + ) + .await; + diesel::sql_query("ROLLBACK") + .execute(&mut conn) + .await + .unwrap(); + assert!(clear.unwrap().is_err()); + assert!(save.unwrap().is_err()); + } + store.clear_pending(second, &lease).await.unwrap(); + assert_eq!(pending_accounts().await.len(), 2); + + let tail = run( + &checking(&[second, joined_later]), + &store, + network, + &lease, + &CancellationToken::new(), + first, + later(), + ) + .await + .unwrap(); + assert_eq!((tail.checked, tail.unresolved), (2, 2)); + assert_eq!(tail.last_checked.as_deref(), Some(joined_later)); + assert_eq!(pending_accounts().await.len(), 3); + assert_eq!(store.remaining_candidates(network, first).await.unwrap(), 2); + + let unreachable = { + let chain = (0..3).fold( + crate::testing::mocks::MockNetworkClient::new(), + |chain, _| chain.with_verify_commitment(Err("simulated RPC outage".into())), + ); + create_test_app_state_with_mocks(storage.clone(), Arc::new(chain), metadata.clone()) + }; + let outage = run( + &unreachable, + &store, + network, + &lease, + &CancellationToken::new(), + "", + later(), + ) + .await + .unwrap(); + assert_eq!((outage.checked, outage.failed), (3, 3)); + assert!(outage.stopped_early); + assert_eq!(outage.last_checked, None); + + let flaky = { + let chain = crate::testing::mocks::MockNetworkClient::new() + .with_verify_commitment(Err("simulated RPC error".into())) + .with_verify_commitment(Ok(chain_head(second))) + .with_verify_commitment(Err("simulated RPC error".into())); + create_test_app_state_with_mocks(storage.clone(), Arc::new(chain), metadata.clone()) + }; + let interleaved = run( + &flaky, + &store, + network, + &lease, + &CancellationToken::new(), + "", + later(), + ) + .await + .unwrap(); + assert_eq!((interleaved.checked, interleaved.failed), (3, 2)); + assert!(!interleaved.stopped_early); + assert_eq!(interleaved.last_checked.as_deref(), Some(joined_later)); + + assert_eq!( + store.progress(network).await.unwrap(), + ScanProgress::default() + ); + let saved = scan_progress(first); + store.save_progress(network, &saved, &lease).await.unwrap(); + assert_eq!(store.progress(network).await.unwrap(), saved); + store.leader.release(lease.clone()).await.unwrap(); + assert!(store.save_progress(network, &saved, &lease).await.is_err()); + + let stopped = RecoveryStore::new(metadata.pool_handle(), None, "stopped-recovery"); + stopped + .leader + .try_acquire(Duration::from_secs(60)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + run_within( + &checking(&[]), + &store, + network, + Duration::ZERO, + Duration::ZERO + ) + .await, + Err(StartupRecoveryError::LeaseBusy) + )); + let waited_from = Instant::now(); + assert!(matches!( + run_within( + &checking(&[]), + &store, + network, + Duration::ZERO, + Duration::from_secs(3) + ) + .await, + Err(StartupRecoveryError::LeaseBusy) + )); + assert!(waited_from.elapsed() >= LEASE_RETRY_INTERVAL); + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "UPDATE worker_leases SET expires_at = clock_timestamp() + interval '1 second' + WHERE lease_name = 'account_recovery'", + ) + .execute(&mut conn) + .await + .unwrap(); + drop(conn); + run_within( + &checking(&[]), + &store, + network, + Duration::ZERO, + Duration::from_secs(10), + ) + .await + .unwrap(); + assert_eq!(store.progress(network).await.unwrap(), saved); + + run_within( + &checking(&[second, joined_later]), + &store, + network, + Duration::from_secs(60), + Duration::ZERO, + ) + .await + .unwrap(); + assert_eq!( + store.progress(network).await.unwrap(), + ScanProgress::default() + ); + assert!(!store.startup_completed(network).await.unwrap()); + + run_within( + &checking(&[first, second, joined_later]), + &store, + network, + Duration::from_secs(60), + Duration::ZERO, + ) + .await + .unwrap(); + assert!(store.startup_completed(network).await.unwrap()); + + #[derive(diesel::QueryableByName)] + struct Completion { + #[diesel(sql_type = BigInt)] + unresolved_accounts: i64, + } + let mut conn = metadata.pool_handle().get().await.unwrap(); + let completion = diesel::sql_query( + "SELECT unresolved_accounts FROM account_recovery_startup + WHERE network_config = $1 AND completed_at IS NOT NULL", + ) + .bind::( + serde_json::to_value(NetworkConfig::Miden { + network_type: network, + }) + .unwrap(), + ) + .get_result::(&mut conn) + .await + .unwrap(); + assert_eq!(completion.unresolved_accounts, 3); + + diesel::sql_query("UPDATE states SET nonce = 4 WHERE account_id = $1") + .bind::(joined_later) + .execute(&mut conn) + .await + .unwrap(); + let same_nonce = store + .candidates(network, second, 10) + .await + .unwrap() + .pop() + .unwrap(); + assert_eq!( + (same_nonce.stored_nonce, same_nonce.canonical_nonce), + (Some(4), Some(4)) + ); + let lease = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + assert!( + store + .apply( + &same_nonce, + &serde_json::json!({"restored": true}), + &Auth::MidenEcdsa { + cosigner_commitments: vec![Word::from([1, 2, 3, 4_u32]).into_hex()], + }, + chrono::Utc::now(), + &lease, + ) + .await + .unwrap() + ); + let restored = storage.pull_state_head(joined_later).await.unwrap(); + assert_eq!( + (restored.commitment.as_str(), restored.nonce), + ("0xheld-head", Some(4)) + ); + store.leader.release(lease).await.unwrap(); +} diff --git a/crates/server/src/jobs/mod.rs b/crates/server/src/jobs/mod.rs index df1728d8..21477e5a 100644 --- a/crates/server/src/jobs/mod.rs +++ b/crates/server/src/jobs/mod.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "postgres")] +pub(crate) mod account_recovery; pub mod block_watch; pub mod canonicalization; pub mod execution_reconcile; diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index d609e513..8b29a5d0 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -10,10 +10,32 @@ use server::network::{NetworkType, RpcSettings}; use std::env; use std::path::PathBuf; +#[cfg(feature = "postgres")] +use server::builder::handle::StartupRecoveryError; + +const STARTUP_RECOVERY_ENV: &str = "GUARDIAN_STARTUP_ACCOUNT_RECOVERY"; + #[tokio::main] async fn main() { dotenvy::dotenv().ok(); + let startup_recovery = match env::var(STARTUP_RECOVERY_ENV) { + Ok(value) => value.trim().parse::().unwrap_or_else(|_| { + eprintln!("{STARTUP_RECOVERY_ENV} must be true or false"); + std::process::exit(2); + }), + Err(env::VarError::NotPresent) => false, + Err(env::VarError::NotUnicode(_)) => { + eprintln!("{STARTUP_RECOVERY_ENV} must be valid UTF-8"); + std::process::exit(2); + } + }; + #[cfg(not(feature = "postgres"))] + if startup_recovery { + eprintln!("Account recovery requires a Postgres build"); + std::process::exit(2); + } + let keystore_path: PathBuf = env::var("GUARDIAN_KEYSTORE_PATH") .unwrap_or_else(|_| "/var/guardian/keystore".to_string()) .into(); @@ -39,7 +61,7 @@ async fn main() { let release_sweep = server::release_sweep::ReleaseSweepConfig::from_env() .expect("Invalid release sweep configuration"); - ServerBuilder::new() + let server = ServerBuilder::new() .with_logging(LoggingConfig::default()) .network(network_type) .with_rpc(RpcSettings::from_env(network_type).expect("Invalid RPC configuration")) @@ -81,7 +103,24 @@ async fn main() { .cors(cors_layer) .build() .await - .expect("Failed to build server") - .run() - .await; + .expect("Failed to build server"); + + #[cfg(feature = "postgres")] + if startup_recovery { + match server.recover_accounts_on_startup().await { + Ok(()) => {} + Err(error @ StartupRecoveryError::Incomplete(_)) => { + tracing::error!(%error, "Startup account recovery did not finish; serving without it"); + } + Err( + error @ (StartupRecoveryError::LeaseBusy + | StartupRecoveryError::LeaseUnavailable(_)), + ) => { + tracing::error!(%error, "Cannot serve while account recovery may be active elsewhere"); + std::process::exit(1); + } + } + } + + server.run().await; } diff --git a/crates/server/src/schema.rs b/crates/server/src/schema.rs index dd7d0df8..3aee4820 100644 --- a/crates/server/src/schema.rs +++ b/crates/server/src/schema.rs @@ -1,5 +1,49 @@ // @generated automatically by Diesel CLI. +diesel::table! { + /// Representation of the `account_recovery_startup` table. + account_recovery_startup (network_config) { + network_config -> Jsonb, + #[max_length = 128] + resume_after -> Varchar, + completed_at -> Nullable, + unresolved_accounts -> Nullable, + } +} + +diesel::table! { + /// Representation of the `account_recovery_pending` table. + account_recovery_pending (account_id) { + #[max_length = 128] + account_id -> Varchar, + outcome -> Text, + reason -> Text, + #[max_length = 128] + stored_commitment -> Varchar, + canonical_nonce -> Nullable, + #[max_length = 128] + canonical_commitment -> Nullable, + history_starts_at_nonce -> Nullable, + #[max_length = 128] + history_base_commitment -> Nullable, + recorded_at -> Timestamptz, + } +} + +diesel::table! { + /// Representation of the `account_recovery_backups` table. + account_recovery_backups (id) { + id -> Int8, + #[max_length = 128] + account_id -> Varchar, + prior_state -> Jsonb, + prior_auth -> Jsonb, + #[max_length = 128] + recovered_commitment -> Varchar, + recovered_at -> Timestamptz, + } +} + diesel::table! { /// Representation of the `states` table. /// diff --git a/crates/server/src/storage/postgres.rs b/crates/server/src/storage/postgres.rs index 5b69e845..6f2d7653 100644 --- a/crates/server/src/storage/postgres.rs +++ b/crates/server/src/storage/postgres.rs @@ -36,6 +36,8 @@ use url::Url; #[path = "postgres_execution.rs"] mod execution; +#[path = "postgres_recovery.rs"] +pub(crate) mod recovery; pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); diff --git a/crates/server/src/storage/postgres_recovery.rs b/crates/server/src/storage/postgres_recovery.rs new file mode 100644 index 00000000..57a94892 --- /dev/null +++ b/crates/server/src/storage/postgres_recovery.rs @@ -0,0 +1,526 @@ +use std::{sync::Arc, time::Duration}; + +use chrono::{DateTime, Utc}; +use diesel::prelude::*; +use diesel::sql_types::{BigInt, Bool, Jsonb, Nullable, Text, Timestamptz}; +use diesel_async::scoped_futures::{ScopedBoxFuture, ScopedFutureExt}; +use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl, SimpleAsyncConnection}; +use serde_json::Value; +use tokio::time::timeout; + +use crate::coordination::{LeaderElector, Lease, postgres::PgLeaseElector}; +use crate::metadata::{Auth, MidenNetworkType, NetworkConfig}; +use crate::schema::{account_metadata, states}; +use crate::storage::LeaseFence; +use crate::storage::encryption::{cipher::StorageCipher, envelope::RecordAad}; + +use super::{Pool, lease_fence_is_current, lock_account_metadata}; + +const UNSETTLED_ACCOUNT_WORK: &str = " + CASE + WHEN m.paused_at IS NOT NULL THEN 'the account is paused' + WHEN m.has_pending_candidate OR EXISTS ( + SELECT 1 FROM deltas pending WHERE pending.account_id = st.account_id + AND pending.status_kind IN ('candidate', 'retained') + ) THEN 'a delta awaits its on-chain outcome' + WHEN EXISTS ( + SELECT 1 FROM deltas abandoned WHERE abandoned.account_id = st.account_id + AND abandoned.status_kind = 'discarded' + AND abandoned.status->>'reason' = 'client_abandoned' + ) THEN 'a client-abandoned delta is on record' + WHEN EXISTS (SELECT 1 FROM superseded_deltas d WHERE d.account_id = st.account_id) + THEN 'a superseded delta is retained' + WHEN EXISTS ( + SELECT 1 FROM delta_proposals p WHERE p.account_id = st.account_id AND p.status_kind = 'pending' + ) THEN 'a proposal is pending' + WHEN EXISTS ( + SELECT 1 FROM execution_reservations r WHERE r.account_id = st.account_id AND r.released_at IS NULL + ) THEN 'an execution is reserved' + END"; + +const CANDIDATE_SOURCE: &str = " + FROM account_metadata m + JOIN states st USING (account_id) + LEFT JOIN LATERAL ( + SELECT nonce, new_commitment FROM deltas + WHERE account_id = st.account_id AND status_kind = 'canonical' + ORDER BY nonce DESC LIMIT 1 + ) d ON TRUE + LEFT JOIN LATERAL ( + SELECT nonce, prev_commitment FROM deltas + WHERE account_id = st.account_id AND status_kind = 'canonical' + ORDER BY nonce LIMIT 1 + ) first_held ON TRUE + WHERE m.account_id > $1 AND m.network_config->>'kind' = 'miden' + AND (m.network_config = $2 OR m.network_config->>'network_type' IN ('local', 'devnet')) + AND m.released_at IS NULL + AND (d.nonce IS NOT NULL OR st.updated_at > st.created_at) + AND d.new_commitment IS DISTINCT FROM st.commitment"; + +const HELD_LEASE: &str = " + FROM worker_leases + WHERE lease_name = $1 AND holder_id = $2 AND fence_token = $3 + AND clock_timestamp() < expires_at"; + +#[derive(diesel::QueryableByName)] +pub(crate) struct RecoveryCandidate { + #[diesel(sql_type = Text)] + pub account_id: String, + #[diesel(sql_type = Nullable)] + pub stored_nonce: Option, + #[diesel(sql_type = Text)] + pub stored_commitment: String, + #[diesel(sql_type = Nullable)] + pub canonical_nonce: Option, + #[diesel(sql_type = Nullable)] + pub canonical_commitment: Option, + #[diesel(sql_type = Nullable)] + pub unsettled_work: Option, + #[diesel(sql_type = Nullable)] + pub history_starts_at_nonce: Option, + #[diesel(sql_type = Nullable)] + pub history_base_commitment: Option, + #[diesel(sql_type = Bool)] + pub legacy_network_label: bool, + #[diesel(sql_type = Timestamptz)] + state_updated_at: DateTime, + #[diesel(sql_type = BigInt)] + auth_version: i64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PendingOutcome { + Deferred, + Unresolved, +} + +impl PendingOutcome { + fn label(self) -> &'static str { + match self { + Self::Deferred => "deferred", + Self::Unresolved => "unresolved", + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, diesel::QueryableByName)] +pub(crate) struct ScanProgress { + #[diesel(sql_type = Text)] + pub resume_after: String, +} + +pub(crate) struct RecoveryStore { + pool: Pool, + cipher: Option>, + pub leader: Arc, +} + +impl RecoveryStore { + pub(crate) fn new( + pool: Pool, + cipher: Option>, + holder_id: impl Into, + ) -> Self { + let leader = Arc::new(PgLeaseElector::new( + pool.clone(), + "account_recovery", + holder_id, + )); + Self { + pool, + cipher, + leader, + } + } + + pub(crate) async fn startup_completed( + &self, + network_type: MidenNetworkType, + ) -> Result { + #[derive(diesel::QueryableByName)] + struct Completion { + #[diesel(sql_type = Bool)] + completed: bool, + } + + let network = miden_network_config(network_type)?; + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query( + "SELECT EXISTS ( + SELECT 1 FROM account_recovery_startup + WHERE network_config = $1 AND unresolved_accounts IS NOT NULL + ) AS completed", + ) + .bind::(network) + .get_result::(&mut conn) + .await + .map(|row| row.completed) + .map_err(|error| format!("Could not read startup recovery completion: {error}")) + } + + pub(crate) async fn progress( + &self, + network_type: MidenNetworkType, + ) -> Result { + let network = miden_network_config(network_type)?; + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query( + "SELECT resume_after FROM account_recovery_startup WHERE network_config = $1", + ) + .bind::(network) + .get_result::(&mut conn) + .await + .optional() + .map(Option::unwrap_or_default) + .map_err(|error| format!("Could not read startup recovery progress: {error}")) + } + + pub(crate) async fn save_progress( + &self, + network_type: MidenNetworkType, + progress: &ScanProgress, + lease: &Lease, + ) -> Result<(), String> { + let network = miden_network_config(network_type)?; + let written = self + .write("record startup recovery progress", |conn| { + async move { + diesel::sql_query(format!( + "INSERT INTO account_recovery_startup (network_config, resume_after) + SELECT $4, $5 {HELD_LEASE} + ON CONFLICT (network_config) DO UPDATE + SET resume_after = EXCLUDED.resume_after + WHERE account_recovery_startup.unresolved_accounts IS NULL", + )) + .bind::(&lease.name) + .bind::(&lease.holder_id) + .bind::(lease.fence_token) + .bind::(network) + .bind::(&progress.resume_after) + .execute(conn) + .await + } + .scope_boxed() + }) + .await?; + if written == 0 { + return Err("Account recovery lost its lease before recording scan progress".into()); + } + Ok(()) + } + + pub(crate) async fn complete_startup( + &self, + network_type: MidenNetworkType, + unresolved_accounts: i64, + lease: &Lease, + ) -> Result<(), String> { + let network = miden_network_config(network_type)?; + let written = self.write("record startup recovery completion", |conn| async move { + diesel::sql_query(format!( + "INSERT INTO account_recovery_startup (network_config, unresolved_accounts, completed_at) + SELECT $4, $5, now() {HELD_LEASE} + ON CONFLICT (network_config) DO UPDATE + SET completed_at = EXCLUDED.completed_at, + unresolved_accounts = EXCLUDED.unresolved_accounts, + resume_after = '' + WHERE account_recovery_startup.unresolved_accounts IS NULL", + )) + .bind::(&lease.name) + .bind::(&lease.holder_id) + .bind::(lease.fence_token) + .bind::(network) + .bind::(unresolved_accounts) + .execute(conn) + .await + }.scope_boxed()).await?; + if written == 0 { + return Err( + "Account recovery lost its lease before recording startup completion".into(), + ); + } + Ok(()) + } + + pub(crate) async fn candidates( + &self, + network_type: MidenNetworkType, + after: &str, + limit: i64, + ) -> Result, String> { + let network = miden_network_config(network_type)?; + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query(format!( + "SELECT st.account_id, st.nonce AS stored_nonce, st.commitment AS stored_commitment, + d.nonce AS canonical_nonce, ({UNSETTLED_ACCOUNT_WORK}) AS unsettled_work, + d.new_commitment AS canonical_commitment, st.updated_at AS state_updated_at, + m.auth_version, m.network_config <> $2 AS legacy_network_label, + first_held.nonce AS history_starts_at_nonce, + first_held.prev_commitment AS history_base_commitment + {CANDIDATE_SOURCE} + ORDER BY m.account_id + LIMIT $3", + )) + .bind::(after) + .bind::(network) + .bind::(limit) + .load(&mut conn) + .await + .map_err(|error| format!("Could not list recovery candidates: {error}")) + } + + pub(crate) async fn remaining_candidates( + &self, + network_type: MidenNetworkType, + after: &str, + ) -> Result { + #[derive(diesel::QueryableByName)] + struct Remaining { + #[diesel(sql_type = BigInt)] + remaining: i64, + } + + let network = miden_network_config(network_type)?; + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query(format!("SELECT count(*) AS remaining {CANDIDATE_SOURCE}")) + .bind::(after) + .bind::(network) + .get_result::(&mut conn) + .await + .map(|row| row.remaining) + .map_err(|error| format!("Could not count remaining recovery candidates: {error}")) + } + + pub(crate) async fn record_pending( + &self, + candidate: &RecoveryCandidate, + outcome: PendingOutcome, + reason: &str, + lease: &Lease, + ) -> Result<(), String> { + let written = self + .write("record the pending account", |conn| { + async move { + diesel::sql_query(format!( + "INSERT INTO account_recovery_pending + (account_id, outcome, reason, stored_commitment, canonical_nonce, + canonical_commitment, history_starts_at_nonce, history_base_commitment) + SELECT $4, $5, $6, $7, $8, $9, $10, $11 {HELD_LEASE} + ON CONFLICT (account_id) DO UPDATE + SET outcome = EXCLUDED.outcome, + reason = EXCLUDED.reason, + stored_commitment = EXCLUDED.stored_commitment, + canonical_nonce = EXCLUDED.canonical_nonce, + canonical_commitment = EXCLUDED.canonical_commitment, + history_starts_at_nonce = EXCLUDED.history_starts_at_nonce, + history_base_commitment = EXCLUDED.history_base_commitment, + recorded_at = EXCLUDED.recorded_at", + )) + .bind::(&lease.name) + .bind::(&lease.holder_id) + .bind::(lease.fence_token) + .bind::(&candidate.account_id) + .bind::(outcome.label()) + .bind::(reason) + .bind::(&candidate.stored_commitment) + .bind::, _>(candidate.canonical_nonce) + .bind::, _>(candidate.canonical_commitment.as_deref()) + .bind::, _>(candidate.history_starts_at_nonce) + .bind::, _>(candidate.history_base_commitment.as_deref()) + .execute(conn) + .await + } + .scope_boxed() + }) + .await?; + if written == 0 { + return Err( + "Account recovery lost its lease before recording the pending account".into(), + ); + } + Ok(()) + } + + pub(crate) async fn clear_pending( + &self, + account_id: &str, + lease: &Lease, + ) -> Result<(), String> { + self.write("clear the pending account", |conn| { + async move { + diesel::sql_query(format!( + "DELETE FROM account_recovery_pending WHERE account_id = $4 + AND EXISTS (SELECT 1 {HELD_LEASE})", + )) + .bind::(&lease.name) + .bind::(&lease.holder_id) + .bind::(lease.fence_token) + .bind::(account_id) + .execute(conn) + .await + .map(drop) + } + .scope_boxed() + }) + .await + } + + pub(crate) async fn apply( + &self, + candidate: &RecoveryCandidate, + state_json: &Value, + auth: &Auth, + now: DateTime, + lease: &Lease, + ) -> Result { + let account_id = &candidate.account_id; + let canonical_nonce = candidate + .canonical_nonce + .ok_or("Recovery target has no canonical nonce")?; + let canonical_commitment = candidate + .canonical_commitment + .as_deref() + .ok_or("Recovery target has no canonical commitment")?; + let stored_json = match &self.cipher { + Some(cipher) => cipher + .encrypt(&RecordAad::State { account_id }, state_json) + .map_err(|error| error.to_string())?, + None => state_json.clone(), + }; + let auth_json = serde_json::to_value(auth).map_err(|error| error.to_string())?; + self.write("restore the account", |conn| { + async move { + if lock_account_metadata(conn, account_id) + .await + .optional()? + .is_none() + || !candidate + .lock_if_eligible(conn, canonical_nonce, canonical_commitment) + .await? + { + return Ok(false); + } + + let fence = LeaseFence { + lease_name: lease.name.clone(), + holder_id: lease.holder_id.clone(), + fence_token: lease.fence_token, + }; + if !lease_fence_is_current(conn, &fence).await? { + return Ok(false); + } + + diesel::sql_query( + "INSERT INTO account_recovery_backups + (account_id, prior_state, prior_auth, recovered_commitment) + SELECT st.account_id, to_jsonb(st), m.auth, $2 + FROM states st JOIN account_metadata m USING (account_id) + WHERE st.account_id = $1", + ) + .bind::(account_id) + .bind::(canonical_commitment) + .execute(conn) + .await?; + + let written_nonce = diesel::update(states::table.find(account_id)) + .set(( + states::commitment.eq(canonical_commitment), + states::state_json.eq(stored_json), + states::nonce.eq(Some(canonical_nonce)), + states::updated_at.eq(now), + )) + .returning(states::nonce) + .get_result::>(conn) + .await?; + if written_nonce.is_none() { + diesel::update(states::table.find(account_id)) + .set(states::nonce.eq(Some(canonical_nonce))) + .execute(conn) + .await?; + } + + diesel::update(account_metadata::table.find(account_id)) + .set(( + account_metadata::auth.eq(auth_json), + account_metadata::updated_at.eq(now), + )) + .execute(conn) + .await?; + Ok(true) + } + .scope_boxed() + }) + .await + } + + async fn write<'a, T, F>(&self, operation: &str, work: F) -> Result + where + T: Send + 'a, + F: for<'conn> FnOnce( + &'conn mut AsyncPgConnection, + ) -> ScopedBoxFuture<'a, 'conn, QueryResult> + + Send + + 'a, + { + let mut conn = timeout(Duration::from_secs(3), self.pool.get()) + .await + .map_err(|_| format!("Timed out waiting for a connection to {operation}"))? + .map_err(|error| format!("Could not get a connection to {operation}: {error}"))?; + AsyncConnection::transaction(&mut *conn, |conn| { + async move { + conn.batch_execute( + "SET LOCAL statement_timeout = '2s'; SET LOCAL lock_timeout = '2s';", + ) + .await?; + work(conn).await + } + .scope_boxed() + }) + .await + .map_err(|error| format!("Could not {operation}: {error}")) + } +} + +fn miden_network_config(network_type: MidenNetworkType) -> Result { + serde_json::to_value(NetworkConfig::Miden { network_type }).map_err(|error| error.to_string()) +} + +#[derive(diesel::QueryableByName)] +struct LockedState { + #[diesel(sql_type = Text, column_name = account_id)] + _account_id: String, +} + +impl RecoveryCandidate { + async fn lock_if_eligible( + &self, + conn: &mut AsyncPgConnection, + canonical_nonce: i64, + canonical_commitment: &str, + ) -> QueryResult { + diesel::sql_query(format!( + "SELECT st.account_id + FROM states st JOIN account_metadata m USING (account_id) + WHERE st.account_id = $1 AND st.nonce IS NOT DISTINCT FROM $2 AND st.commitment = $3 + AND st.updated_at = $4 AND m.auth_version = $5 + AND m.released_at IS NULL AND ({UNSETTLED_ACCOUNT_WORK}) IS NULL + AND ($6, $7) = ( + SELECT d.nonce, d.new_commitment FROM deltas d + WHERE d.account_id = st.account_id AND d.status_kind = 'canonical' + ORDER BY d.nonce DESC LIMIT 1 + ) + FOR UPDATE OF st", + )) + .bind::(&self.account_id) + .bind::, _>(self.stored_nonce) + .bind::(&self.stored_commitment) + .bind::(self.state_updated_at) + .bind::(self.auth_version) + .bind::(canonical_nonce) + .bind::(canonical_commitment) + .get_result::(conn) + .await + .optional() + .map(|row| row.is_some()) + } +} diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 39fc8a0e..d7b98ae7 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -30,6 +30,7 @@ the runtime env vars in this document. | `GUARDIAN_KEYSTORE_PATH` | `/var/guardian/keystore` | any | Local Falcon/ECDSA key files (ACK signers and per-account creds). | | `GUARDIAN_DB_POOL_MAX_SIZE` | `16`; `32` when `GUARDIAN_ENV=prod` | `postgres` | Storage backend pool size. | | `GUARDIAN_METADATA_DB_POOL_MAX_SIZE` | matches storage | `postgres` | Metadata backend pool size; usually leave equal. | +| `GUARDIAN_STARTUP_ACCOUNT_RECOVERY` | `false` | `postgres` | Checks recovery candidates for up to 5 minutes before starting API listeners. Later startups continue where the scan stopped. Use with stop-before-start upgrades and one replica. Guardian skips recovery when `GUARDIAN_MAX_REPLICAS` exceeds `1`. If another process holds the recovery lease, or the server cannot confirm it holds it alone, the server exits before opening its listeners. AWS deployments set it through Terraform variable `guardian_startup_account_recovery`, which also lengthens the task health check timings. See [Startup account recovery](./PRODUCTION.md#startup-account-recovery). | | `GUARDIAN_METADATA_CACHE_MAX_BYTES` | `268435456` (256 MiB) | `postgres` | Memory bound of the per-replica account metadata cache used by read routes. Every read still confirms the copy is current against the database (one indexed lookup of a version the database raises on every metadata update), so the cache never serves stale authorization data; it only saves loading and parsing the row. `0` disables caching. | | `GUARDIAN_CANONICALIZATION_FAST_PROMOTION_ENABLED` | `true` | any | Enables the additional promotion-only pass for recent candidates. Set to `false` to use only the full canonicalization interval; AWS deployments can set Terraform variable `guardian_canonicalization_fast_promotion_enabled = false`. | | `GUARDIAN_CANONICALIZATION_MAX_CONCURRENT_ACCOUNTS` | `10`; `50` when `GUARDIAN_ENV=prod` | any | Accounts one canonicalization pass processes in parallel; `1` = fully sequential. Each account holds a DB connection only during its short fenced transactions (the dominant cost is a connectionless chain RPC), so this may exceed `GUARDIAN_DB_POOL_MAX_SIZE`; simultaneous write bursts just queue briefly at the pool. | diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index d087315a..8347f3f1 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -350,6 +350,104 @@ Two Guardian-specific caveats when restoring from a backup: The verification and restore procedure is in [`runbooks/backup-restore.md`](./runbooks/backup-restore.md). +### Startup account recovery + +Guardian can check its stored Miden account records against the Miden network +when it starts, and bring a record back in line with the account's canonical +history when the two disagree. The check is off by default. + +**Enabling it** + +- Set `GUARDIAN_STARTUP_ACCOUNT_RECOVERY=true` on a single Guardian instance + backed by Postgres. With the bundled Compose files, set it in `.env`. +- Stop the old Guardian process before the new one starts. Rolling + deployments overlap old and new servers even with a target of one instance, + so keep the flag off for those. +- Guardian skips the check when `GUARDIAN_MAX_REPLICAS` exceeds `1`. +- On the AWS stack in `infra/`, set Terraform variable + `guardian_startup_account_recovery = true` for that one deploy. It passes + the flag to the task and lengthens the container health check start period + and the load balancer grace period, so the task is not replaced while its + listeners are closed. Running a single task (desired count and autoscaling + maximum of one) and stopping the running task before the deploy remain + manual steps. Set the variable back to `false` once the check has completed. + +**What it does** + +The check runs before the API listeners open. For each account whose stored +record differs from its latest canonical delta, Guardian asks the configured +Miden node for the account's current commitment. + +- If the stored record already matches the network, nothing changes. +- If Guardian holds the account's canonical deltas from creation (up to 128), + it rebuilds the account from them. When the result matches the network and + is bound to this server's ACK key, Guardian stores it together with its + nonce and the account metadata derived from it. The same transaction saves + the previous record in `account_recovery_backups`; encrypted state keeps its + original ciphertext. +- Otherwise the account is left unchanged and listed in + `account_recovery_pending`. + +A paused account, or one with work in progress such as a delta awaiting its +outcome, a pending proposal or a reserved execution, is deferred and checked +again on a later startup. + +**How long it takes** + +- Guardian checks accounts for up to 5 minutes and lets a write in progress + finish. The listeners can stay closed for about 6 minutes: the 5-minute + budget, up to 35 seconds waiting for the recovery lease, and the final + writes. Allow for that in any startup health check or grace period. +- The check stops for this startup after three account checks in a row fail, + for example while the Miden node is unreachable. +- Guardian records where it stopped in + `account_recovery_startup.resume_after`, and the next startup with the flag + on continues from there. +- The check completes when one startup covers every account, from the first + to the last, without deferring any. Guardian then records how many accounts + it could not bring up to date in + `account_recovery_startup.unresolved_accounts`, and later startups skip the + check. Until then, each startup with the flag on runs it again. +- An account Guardian holds canonical deltas for, but the configured node has + no record of, is deferred. It keeps the check open until the node knows the + account or the account is released. + +**If it cannot run** + +Guardian exits before opening its listeners if another process holds the +recovery lease, or if it cannot confirm it holds the lease alone. It first +waits up to 35 seconds for a lease a stopped process left behind. Any other +error is logged, and Guardian opens its listeners without finishing the check. + +**Reviewing the result** + +`account_recovery_pending` has one row for each account that was checked and +not brought up to date. `outcome` is `deferred` (a later startup checks it +again) or `unresolved` (Guardian does not hold what it needs), and `reason` +says why. When the canonical history held here does not reach back to the +account's creation, `history_starts_at_nonce` and `history_base_commitment` +show where it begins. An unresolved account is brought up to date the next +time it is registered with its current state. + +A row is removed once its account is brought up to date or found unchanged. A +row whose `stored_commitment` no longer matches the account's stored state is +out of date. Export the current list with: + +```sql +SELECT jsonb_agg(to_jsonb(p) ORDER BY p.account_id) +FROM account_recovery_pending p +JOIN states st ON st.account_id = p.account_id + AND st.commitment = p.stored_commitment; +``` + +To run the check again after it completed, remove its completion record and +restart Guardian: + +```sql +DELETE FROM account_recovery_startup +WHERE network_config->>'network_type' = ''; +``` + ## Storage encryption Guardian can encrypt the sensitive stored payloads (account state, delta and diff --git a/infra/README.md b/infra/README.md index b8e22f28..0d57ad80 100644 --- a/infra/README.md +++ b/infra/README.md @@ -116,6 +116,7 @@ server_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/guardian-server # guardian_canonicalization_fast_promotion_enabled = false # guardian_max_pending_candidates_per_account = 4 # opt in to queueing chained candidates (server default 1, max 16; upgrade TS clients first, see docs/CONFIGURATION.md) # guardian_release_sweep_enabled = false # turns the release sweep's walk of the fleet off +# guardian_startup_account_recovery = true # one deploy only: single task, older task stopped first; see docs/PRODUCTION.md # guardian_block_watch_enabled = false # kill switch for the committed-block feed # Optional: dashboard operator Falcon public keys managed by Terraform @@ -303,6 +304,7 @@ aws ecr delete-repository --repository-name "$ECR_REPO_NAME" --force --region "$ | `guardian_canonicalization_fast_promotion_enabled` | `true` | Enables the recent-candidate promotion-only pass in the ECS task definition | | `guardian_max_pending_candidates_per_account` | server default (`1`) | `GUARDIAN_MAX_PENDING_CANDIDATES_PER_ACCOUNT`: chained candidate deltas one account may hold in flight (issue #17). `1` keeps one in-flight candidate per account; `2` to `16` opts in to queueing back-to-back transactions, after TypeScript clients are upgraded (see [`docs/CONFIGURATION.md`](../docs/CONFIGURATION.md)) | | `guardian_release_sweep_enabled` | `true` | Runs the release sweep's walk of the fleet (the rotation); `false` turns the walk off, and block watch too unless `guardian_block_watch_enabled` is set | +| `guardian_startup_account_recovery` | `false` | `GUARDIAN_STARTUP_ACCOUNT_RECOVERY`: runs account recovery before the listeners open, and lengthens the container health check start period (to 300 s) and the load balancer grace period (to 420 s) so the task is not replaced during the scan. Needs a single task and a deploy with no older task still running; turn it off again once the scan has completed. See [Startup account recovery](../docs/PRODUCTION.md#startup-account-recovery) | | `guardian_release_sweep_rotation_seconds` | server default (`21600`) | Target time for one release sweep walk of the fleet; `604800` (weekly) is enough once block watch is healthy | | `guardian_block_watch_enabled` | unset (follows `guardian_release_sweep_enabled`) | Runs block watch (issue #585): switch detection from the node's committed-block feed; `false` is its kill switch, `true` runs it even with the release sweep's walk turned off | | `guardian_block_watch_transport` | `auto` | `GUARDIAN_BLOCK_WATCH_TRANSPORT`: `auto` (the block stream, polling when it is unavailable), `stream`, or `poll` | diff --git a/infra/data.tf b/infra/data.tf index e78c6200..da3fe7c4 100644 --- a/infra/data.tf +++ b/infra/data.tf @@ -166,6 +166,14 @@ locals { (local.effective_server_memory - local.fargate_memory_by_cpu[tostring(local.effective_server_cpu)].min) % local.fargate_memory_by_cpu[tostring(local.effective_server_cpu)].step == 0, false ) + # Account recovery keeps the listeners closed for about 6 minutes at most: + # a 5-minute scan budget, up to 35 seconds waiting for a lease a stopped + # task left behind, and the writes in progress when the budget ends. 300 is + # the longest container start period ECS accepts; the health check's three + # retries at 30 seconds cover the remainder. + server_health_check_start_period_seconds = var.guardian_startup_account_recovery ? 300 : 10 + server_health_check_grace_period_seconds = var.guardian_startup_account_recovery ? 420 : 30 + effective_server_desired_count = var.server_desired_count != null ? var.server_desired_count : (local.is_prod ? 2 : 1) effective_server_autoscaling_enabled = var.server_autoscaling_enabled != null ? var.server_autoscaling_enabled : local.is_prod effective_server_autoscaling_min_capacity = var.server_autoscaling_min_capacity != null ? var.server_autoscaling_min_capacity : local.effective_server_desired_count diff --git a/infra/ecs.tf b/infra/ecs.tf index ec42176f..3225248e 100644 --- a/infra/ecs.tf +++ b/infra/ecs.tf @@ -202,6 +202,10 @@ resource "aws_ecs_task_definition" "server" { name = "GUARDIAN_RELEASE_SWEEP_ENABLED" value = tostring(var.guardian_release_sweep_enabled) }, + { + name = "GUARDIAN_STARTUP_ACCOUNT_RECOVERY" + value = tostring(var.guardian_startup_account_recovery) + }, { name = "GUARDIAN_BLOCK_WATCH_TRANSPORT" value = var.guardian_block_watch_transport @@ -422,7 +426,7 @@ resource "aws_ecs_task_definition" "server" { interval = 30 timeout = 5 retries = 3 - startPeriod = 10 + startPeriod = local.server_health_check_start_period_seconds } logConfiguration = { @@ -460,7 +464,7 @@ resource "aws_ecs_service" "server" { } } - health_check_grace_period_seconds = 30 + health_check_grace_period_seconds = local.server_health_check_grace_period_seconds network_configuration { subnets = local.subnet_ids diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 2a31543e..975c6c47 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -130,6 +130,7 @@ server_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/guardian-server # guardian_canonicalization_fast_promotion_enabled = false # guardian_max_pending_candidates_per_account = 4 # opt in to queueing chained candidates per account (server default 1, max 16) # guardian_release_sweep_enabled = false # turns the release sweep's walk of the fleet off +# guardian_startup_account_recovery = true # one deploy only: single task, older task stopped first; see docs/PRODUCTION.md # guardian_release_sweep_rotation_seconds = 21600 # 604800 (weekly) once block watch is healthy # guardian_release_sweep_max_rate_per_second = 5 # guardian_block_watch_enabled = false # kill switch for the committed-block feed (issue #585); unset follows guardian_release_sweep_enabled diff --git a/infra/variables.tf b/infra/variables.tf index 75ac8df0..b331c2ea 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -570,6 +570,21 @@ variable "guardian_release_sweep_enabled" { default = true } +variable "guardian_startup_account_recovery" { + description = <<-EOT + Whether the server runs account recovery before it opens its listeners + (GUARDIAN_STARTUP_ACCOUNT_RECOVERY). true also lengthens the container + health check start period and the load balancer grace period, so a task + is not replaced while the scan keeps the listeners closed. The operator + arranges the rest: the server skips recovery unless it runs as a single + task, and the deploy must not overlap an older task, so stop the running + task before the deploy that turns this on. Turn it off again once the + scan has completed. + EOT + type = bool + default = false +} + variable "guardian_block_watch_enabled" { description = <<-EOT Whether block watch runs (GUARDIAN_BLOCK_WATCH_ENABLED): the committed-block From cac585b543ef8c4667f45a15c57bc1b1be57579e Mon Sep 17 00:00:00 2001 From: Zeljko Date: Thu, 19 Nov 2026 16:54:11 +0100 Subject: [PATCH 2/8] feat(server): dry run and proposal review for startup account recovery GUARDIAN_STARTUP_ACCOUNT_RECOVERY now takes off, dry-run or apply instead of a boolean. - dry-run performs the same checks, logs what apply would do with each account and writes nothing. It takes no lease. - A restore reviews the account's pending proposals against the signers of the restored state. A proposal one of those signers proposed or approved stays and keeps only their approvals; any other pending proposal is removed. Every proposal a restore changes or removes is saved in account_recovery_backups.prior_proposals. - A pending proposal, a client-abandoned delta or an archived delta no longer defers an account. A pause, a delta awaiting its on-chain outcome and a reserved execution still do. - The scan stops for a startup after three failed checks in a row and resumes with those accounts. It waits up to 35 seconds for a lease a stopped process left behind, and logs when it completes or skips. - infra: guardian_startup_account_recovery is a string with the same three values. Any value but off lengthens the task health check timings. The proposal review follows the approach of #606. --- .../up.sql | 1 + crates/server/src/builder/handle.rs | 43 ++- crates/server/src/jobs/account_recovery.rs | 213 +++++++++++---- .../jobs/account_recovery/postgres_tests.rs | 255 +++++++++++++++++- .../src/jobs/account_recovery/proposals.rs | 67 +++++ crates/server/src/main.rs | 19 +- crates/server/src/schema.rs | 1 + .../server/src/storage/postgres_recovery.rs | 113 +++++++- docs/CONFIGURATION.md | 2 +- docs/PRODUCTION.md | 68 +++-- infra/README.md | 4 +- infra/data.tf | 4 +- infra/ecs.tf | 2 +- infra/terraform.tfvars.example | 2 +- infra/variables.tf | 26 +- 15 files changed, 686 insertions(+), 134 deletions(-) create mode 100644 crates/server/src/jobs/account_recovery/proposals.rs diff --git a/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql index eeae9278..4b2ac994 100644 --- a/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql +++ b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql @@ -3,6 +3,7 @@ CREATE TABLE account_recovery_backups ( account_id VARCHAR(128) NOT NULL, prior_state JSONB NOT NULL, prior_auth JSONB NOT NULL, + prior_proposals JSONB NOT NULL DEFAULT '[]'::jsonb, recovered_commitment VARCHAR(128) NOT NULL, recovered_at TIMESTAMPTZ NOT NULL DEFAULT now() ); diff --git a/crates/server/src/builder/handle.rs b/crates/server/src/builder/handle.rs index 913b6da6..bf9e8b5f 100644 --- a/crates/server/src/builder/handle.rs +++ b/crates/server/src/builder/handle.rs @@ -70,6 +70,27 @@ pub struct ServerHandle { pub(crate) grpc_port: u16, } +/// Whether the server checks its account records at startup, and whether that check writes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StartupRecoveryMode { + Off, + DryRun, + Apply, +} + +impl std::str::FromStr for StartupRecoveryMode { + type Err = String; + + fn from_str(value: &str) -> Result { + match value.trim() { + "off" => Ok(Self::Off), + "dry-run" => Ok(Self::DryRun), + "apply" => Ok(Self::Apply), + other => Err(format!("must be off, dry-run or apply, got {other:?}")), + } + } +} + #[cfg(feature = "postgres")] #[derive(Debug, thiserror::Error)] pub enum StartupRecoveryError { @@ -91,17 +112,29 @@ impl From for StartupRecoveryError { impl ServerHandle { /// Recover accounts once before a stop-before-start deployment opens its listeners. #[cfg(feature = "postgres")] - pub async fn recover_accounts_on_startup(&self) -> Result<(), StartupRecoveryError> { + pub async fn recover_accounts_on_startup( + &self, + mode: StartupRecoveryMode, + ) -> Result<(), StartupRecoveryError> { + let dry_run = match mode { + StartupRecoveryMode::Off => return Ok(()), + StartupRecoveryMode::DryRun => true, + StartupRecoveryMode::Apply => false, + }; + let (store, network_type) = self + .account_recovery + .as_ref() + .ok_or_else(|| "Account recovery requires shared Postgres storage".to_owned())?; + if dry_run { + return crate::jobs::account_recovery::dry_run(&self.app_state, store, *network_type) + .await; + } if crate::middleware::rate_limit::max_replicas_from_env()? != 1 { tracing::warn!( "Skipping startup account recovery because GUARDIAN_MAX_REPLICAS exceeds 1" ); return Ok(()); } - let (store, network_type) = self - .account_recovery - .as_ref() - .ok_or_else(|| "Account recovery requires shared Postgres storage".to_owned())?; if self.app_state.canonicalization.is_none() { return Err(StartupRecoveryError::Incomplete( "Account recovery requires canonicalization".into(), diff --git a/crates/server/src/jobs/account_recovery.rs b/crates/server/src/jobs/account_recovery.rs index 654ae276..bd876b18 100644 --- a/crates/server/src/jobs/account_recovery.rs +++ b/crates/server/src/jobs/account_recovery.rs @@ -15,7 +15,8 @@ use crate::network::{ use crate::state::AppState; use crate::storage::is_record_corruption; use crate::storage::postgres::recovery::{ - PendingOutcome, RecoveryCandidate, RecoveryStore, ScanProgress, + PendingOutcome, ProposalChange, RecoveryCandidate, RecoveryStore, ReviewedProposal, + ScanProgress, }; const PAGE_SIZE: i64 = 100; @@ -31,20 +32,53 @@ const MAX_CONSECUTIVE_FAILED_CHECKS: usize = 3; #[derive(Debug)] enum RecoveryOutcome { Unchanged, - Verified(Box, Auth), + Verified(Box), Retryable(String), Unresolved(String), } +/// What a restore writes for one account. +#[derive(Debug)] +struct Restoration { + state: AppliedState, + auth: Auth, + proposals: Vec, +} + +impl Restoration { + fn proposals_removed(&self) -> usize { + self.proposals_changed(|change| matches!(change, ProposalChange::Remove)) + } + + fn proposals_trimmed(&self) -> usize { + self.proposals_changed(|change| matches!(change, ProposalChange::DropApprovals(_))) + } + + fn proposals_changed(&self, matches: impl Fn(&ProposalChange) -> bool) -> usize { + self.proposals + .iter() + .filter(|proposal| proposal.change.as_ref().is_some_and(&matches)) + .count() + } +} + enum PendingRecord { Clear, Write(PendingOutcome, String), } +/// Whether a scan writes what it finds. A dry run reads and reports only. +#[derive(Clone, Copy)] +enum ScanWrites<'a> { + Apply(&'a Lease), + DryRun, +} + #[derive(Debug, Default)] struct ScanSummary { checked: usize, restored: usize, + restorable: usize, unchanged: usize, retryable: usize, unresolved: i64, @@ -120,6 +154,27 @@ async fn acquire_lease( } } +/// Runs the scan without a lease and without writing anything: the log lines say what an +/// applying run would do with each account. +pub(crate) async fn dry_run( + state: &AppState, + store: &RecoveryStore, + network_type: MidenNetworkType, +) -> Result<(), StartupRecoveryError> { + run( + state, + store, + network_type, + ScanWrites::DryRun, + &CancellationToken::new(), + "", + Instant::now() + SCAN_BUDGET, + ) + .await + .map(drop) + .map_err(StartupRecoveryError::Incomplete) +} + async fn run_within( state: &AppState, store: &RecoveryStore, @@ -156,7 +211,7 @@ async fn run_within( state, store, network_type, - &lease, + ScanWrites::Apply(&lease), &cancel, &started.resume_after, Instant::now() + budget, @@ -221,12 +276,13 @@ async fn run( state: &AppState, store: &RecoveryStore, network_type: MidenNetworkType, - lease: &Lease, + writes: ScanWrites<'_>, cancel: &CancellationToken, resume_after: &str, deadline: Instant, ) -> Result { - tracing::info!(?network_type, %resume_after, "Starting account recovery scan"); + let dry_run = matches!(writes, ScanWrites::DryRun); + tracing::info!(?network_type, %resume_after, dry_run, "Starting account recovery scan"); let mut after = resume_after.to_owned(); let mut summary = ScanSummary::default(); let mut consecutive_failed_checks = 0; @@ -267,78 +323,99 @@ async fn run( summary.unchanged += 1; PendingRecord::Clear } - Ok(RecoveryOutcome::Verified(applied, auth)) => { - match store - .apply( - &candidate, - &applied.state_json, - &auth, - state.clock.now(), - lease, - ) - .await - { - Ok(true) => { - summary.restored += 1; - tracing::info!(account_id = %candidate.account_id, "Restored chain-verified account state and signers"); - PendingRecord::Clear - } - Ok(false) => { - summary.retryable += 1; - tracing::info!(account_id = %candidate.account_id, "Account became ineligible or changed during recovery"); - PendingRecord::Write( - PendingOutcome::Deferred, - "the account changed or became ineligible during recovery".into(), - ) - } - Err(reason) => { - summary.failed += 1; - tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery transaction failed"); - PendingRecord::Write( - PendingOutcome::Deferred, - "restore transaction failed; see Guardian logs".into(), + Ok(RecoveryOutcome::Verified(restoration)) => match writes { + ScanWrites::DryRun => { + summary.restorable += 1; + tracing::info!( + account_id = %candidate.account_id, + proposals_removed = restoration.proposals_removed(), + proposals_trimmed = restoration.proposals_trimmed(), + "Dry run: the account would be restored" + ); + PendingRecord::Clear + } + ScanWrites::Apply(lease) => { + match store + .apply( + &candidate, + &restoration.state.state_json, + &restoration.auth, + &restoration.proposals, + state.clock.now(), + lease, ) + .await + { + Ok(true) => { + summary.restored += 1; + tracing::info!( + account_id = %candidate.account_id, + proposals_removed = restoration.proposals_removed(), + proposals_trimmed = restoration.proposals_trimmed(), + "Restored chain-verified account state and signers" + ); + PendingRecord::Clear + } + Ok(false) => { + summary.retryable += 1; + tracing::info!(account_id = %candidate.account_id, "Account became ineligible or changed during recovery"); + PendingRecord::Write( + PendingOutcome::Deferred, + "the account changed or became ineligible during recovery" + .into(), + ) + } + Err(reason) => { + summary.failed += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery transaction failed"); + PendingRecord::Write( + PendingOutcome::Deferred, + "restore transaction failed; see Guardian logs".into(), + ) + } } } - } + }, Ok(RecoveryOutcome::Retryable(reason)) => { summary.retryable += 1; - tracing::info!(account_id = %candidate.account_id, %reason, "Account recovery deferred until a later startup"); + tracing::info!(account_id = %candidate.account_id, %reason, dry_run, "Account recovery deferred until a later startup"); PendingRecord::Write(PendingOutcome::Deferred, reason) } Ok(RecoveryOutcome::Unresolved(reason)) => { summary.unresolved += 1; - tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery needs additional evidence"); + tracing::warn!(account_id = %candidate.account_id, %reason, dry_run, "Account recovery needs additional evidence"); PendingRecord::Write(PendingOutcome::Unresolved, reason) } Err(reason) => { summary.failed += 1; - tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery could not check the candidate"); + tracing::warn!(account_id = %candidate.account_id, %reason, dry_run, "Account recovery could not check the candidate"); PendingRecord::Write( PendingOutcome::Deferred, "account verification failed; see Guardian logs".into(), ) } }; - let recorded = match record { - PendingRecord::Clear => { - bounded_db_call( - "clear a pending recovery record", - store.clear_pending(&candidate.account_id, lease), - ) - .await - } - PendingRecord::Write(outcome, reason) => { - bounded_db_call( - "write a pending recovery record", - store.record_pending(&candidate, outcome, &reason, lease), - ) - .await + if let ScanWrites::Apply(lease) = writes { + let recorded = match record { + PendingRecord::Clear => { + bounded_db_call( + "clear a pending recovery record", + store.clear_pending(&candidate.account_id, lease), + ) + .await + } + PendingRecord::Write(outcome, reason) => { + bounded_db_call( + "write a pending recovery record", + store.record_pending(&candidate, outcome, &reason, lease), + ) + .await + } + }; + if let Err(reason) = recorded { + summary.failed += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery could not update the pending record"); } - }; - if let Err(reason) = recorded { - summary.failed += 1; - tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery could not update the pending record"); } summary.last_checked = Some(candidate.account_id); if consecutive_failed_checks == 0 { @@ -372,8 +449,10 @@ async fn run( Some(0) }; tracing::info!( + dry_run, checked = summary.checked, restored = summary.restored, + restorable = summary.restorable, unchanged = summary.unchanged, retryable = summary.retryable, unresolved = summary.unresolved, @@ -523,7 +602,23 @@ async fn check_candidate( } } - Ok(RecoveryOutcome::Verified(Box::new(applied), auth)) + let pending = match state + .storage + .pull_pending_proposals(&candidate.account_id) + .await + { + Ok(pending) => pending, + Err(reason) if is_record_corruption(&reason) => { + return Ok(RecoveryOutcome::Unresolved(reason)); + } + Err(reason) => return Err(reason), + }; + let proposals = proposals::review(&applied, &pending); + Ok(RecoveryOutcome::Verified(Box::new(Restoration { + state: applied, + auth, + proposals, + }))) } fn prove_recovery( @@ -614,5 +709,7 @@ fn recovered_auth(scheme: SignatureScheme, state: &AppliedState) -> Result| DeltaStatus::Pending { + timestamp: "2026-11-14T12:45:00Z".into(), + proposer_id: proposer.commitment_hex.clone(), + cosigner_sigs: approvals, + }; + let record = |commitment: &str, status: DeltaStatus| ProposalRecord { + account_id: account_id.into(), + commitment: commitment.into(), + proposal: DeltaObject { + account_id: account_id.into(), + delta_payload: payload.clone(), + status, + ..Default::default() + }, + }; + let restored = |scheme: SignatureScheme| AppliedState { + state_json: serde_json::Value::Null, + commitment: "0xrestored".into(), + nonce: Some(1), + cosigners: vec![ChainCosigner { + commitment: owner.commitment_hex.clone(), + scheme: Some(scheme), + }], + guardian_commitment: None, + }; + let records = [ + record("signed", pending(&owner, vec![approval(&owner)])), + record( + "mixed", + pending(&owner, vec![approval(&owner), approval(&other_key)]), + ), + record("approved", pending(&other_key, vec![approval(&owner)])), + record("unrelated", pending(&other_key, vec![approval(&other_key)])), + record("unsigned", pending(&owner, Vec::new())), + ]; + + let changes: Vec> = + proposals::review(&restored(SignatureScheme::Falcon), &records) + .into_iter() + .map(|reviewed| reviewed.change) + .collect(); + assert_eq!( + changes, + [ + None, + Some(ProposalChange::DropApprovals(pending( + &owner, + vec![approval(&owner)] + ))), + None, + Some(ProposalChange::Remove), + None, + ] + ); + + let under_another_scheme = proposals::review(&restored(SignatureScheme::Ecdsa), &records[..1]); + assert_eq!( + under_another_scheme[0].change, + Some(ProposalChange::DropApprovals(pending(&owner, Vec::new()))) + ); +} + +#[test] +fn the_startup_recovery_mode_has_three_values() { + use crate::builder::handle::StartupRecoveryMode; + + assert_eq!("off".parse(), Ok(StartupRecoveryMode::Off)); + assert_eq!(" dry-run ".parse(), Ok(StartupRecoveryMode::DryRun)); + assert_eq!("apply".parse(), Ok(StartupRecoveryMode::Apply)); + assert!("true".parse::().is_err()); +} + #[tokio::test] async fn partial_history_fixture_cannot_rebuild_a_missing_base() { let network = MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal); @@ -342,7 +489,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e &state, &store, MidenNetworkType::Testnet, - &lease, + ScanWrites::Apply(&lease), &CancellationToken::new(), "", Instant::now() + Duration::from_millis(40), @@ -384,7 +531,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e storage.submit_delta(&pending).await.unwrap(); assert!(matches!( check_candidate(&state, &candidate).await.unwrap(), - RecoveryOutcome::Verified(_, _) + RecoveryOutcome::Verified(_) )); assert!( !store @@ -392,6 +539,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e &candidate, &canonical.state_json, &expected_auth, + &[], state.clock.now(), &lease ) @@ -508,6 +656,76 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e assert!(deferred.reason.contains("restore transaction failed")); drop(conn); + let other_key = crate::testing::helpers::TestSigner::new(); + let proposal_payload = serde_json::json!({ + "tx_summary": crate::testing::helpers::create_test_delta_payload(account_id), + }); + let other_approval = crate::delta_object::CosignerSignature { + signature: guardian_shared::ProposalSignature::Falcon { + signature: other_key.sign_word( + crate::services::proposal_signature::proposal_tx_summary(&proposal_payload) + .unwrap() + .to_commitment(), + ), + }, + timestamp: "2026-11-14T12:50:00Z".into(), + signer_id: other_key.commitment_hex.clone(), + }; + let pending_proposal = |nonce: u64, proposer: &str| DeltaObject { + account_id: account_id.clone(), + nonce, + prev_commitment: prior.commitment.clone(), + delta_payload: proposal_payload.clone(), + status: DeltaStatus::Pending { + timestamp: "2026-11-14T12:45:00Z".into(), + proposer_id: proposer.into(), + cosigner_sigs: vec![other_approval.clone()], + }, + ..Default::default() + }; + storage + .submit_delta_proposal( + "0xproposed-by-another-key", + &pending_proposal(4, &other_key.commitment_hex), + ) + .await + .unwrap(); + storage + .submit_delta_proposal( + "0xproposed-by-the-signer", + &pending_proposal(5, &signer.commitment_hex), + ) + .await + .unwrap(); + + let preview = run( + &state, + &store, + MidenNetworkType::Testnet, + ScanWrites::DryRun, + &CancellationToken::new(), + "", + Instant::now() + Duration::from_secs(60), + ) + .await + .unwrap(); + assert_eq!( + (preview.checked, preview.restorable, preview.restored), + (1, 1, 0) + ); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment + ); + assert_eq!( + storage + .pull_pending_proposals(account_id) + .await + .unwrap() + .len(), + 2 + ); + run_once(&state, &store, MidenNetworkType::Testnet) .await .unwrap(); @@ -528,6 +746,14 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e Some(2) ); + let kept = storage.pull_pending_proposals(account_id).await.unwrap(); + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].commitment, "0xproposed-by-the-signer"); + assert!(matches!( + &kept[0].proposal.status, + DeltaStatus::Pending { cosigner_sigs, .. } if cosigner_sigs.is_empty() + )); + let held = contender .leader .try_acquire(LEASE_TTL) @@ -556,6 +782,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e &candidate, &canonical.state_json, &expected_auth, + &[], state.clock.now(), &lease ) @@ -606,9 +833,11 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e prior_state: serde_json::Value, #[diesel(sql_type = Jsonb)] prior_auth: serde_json::Value, + #[diesel(sql_type = Jsonb)] + prior_proposals: serde_json::Value, } let mut conn = metadata.pool_handle().get().await.unwrap(); - let backup = diesel::sql_query("SELECT count(*) OVER () AS total, prior_state, prior_auth FROM account_recovery_backups WHERE account_id = $1") + let backup = diesel::sql_query("SELECT count(*) OVER () AS total, prior_state, prior_auth, prior_proposals FROM account_recovery_backups WHERE account_id = $1") .bind::(account_id).get_result::(&mut conn).await.unwrap(); let old_payload = &backup.prior_state["state_json"]; assert_ne!(old_payload, &prior.state_json); @@ -623,6 +852,17 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e old_auth ); assert_eq!(backup.total, 1); + let saved_proposals: Vec<&str> = backup + .prior_proposals + .as_array() + .unwrap() + .iter() + .map(|proposal| proposal["commitment"].as_str().unwrap()) + .collect(); + assert_eq!( + saved_proposals, + ["0xproposed-by-another-key", "0xproposed-by-the-signer"] + ); #[derive(diesel::QueryableByName)] struct PendingCount { @@ -784,7 +1024,7 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen &checking(&[first, second, joined_later]), &store, network, - &lease, + ScanWrites::Apply(&lease), &CancellationToken::new(), "", later(), @@ -872,7 +1112,7 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen &checking(&[second, joined_later]), &store, network, - &lease, + ScanWrites::Apply(&lease), &CancellationToken::new(), first, later(), @@ -895,7 +1135,7 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen &unreachable, &store, network, - &lease, + ScanWrites::Apply(&lease), &CancellationToken::new(), "", later(), @@ -917,7 +1157,7 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen &flaky, &store, network, - &lease, + ScanWrites::Apply(&lease), &CancellationToken::new(), "", later(), @@ -1065,6 +1305,7 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen &Auth::MidenEcdsa { cosigner_commitments: vec![Word::from([1, 2, 3, 4_u32]).into_hex()], }, + &[], chrono::Utc::now(), &lease, ) diff --git a/crates/server/src/jobs/account_recovery/proposals.rs b/crates/server/src/jobs/account_recovery/proposals.rs new file mode 100644 index 00000000..d876e5fc --- /dev/null +++ b/crates/server/src/jobs/account_recovery/proposals.rs @@ -0,0 +1,67 @@ +use crate::delta_object::{CosignerSignature, DeltaObject, DeltaStatus}; +use crate::metadata::auth::ChainCosigner; +use crate::network::AppliedState; +use crate::services::proposal_signature::{proposal_tx_summary, verify_proposal_signature}; +use crate::storage::ProposalRecord; +use crate::storage::postgres::recovery::{ProposalChange, ReviewedProposal}; + +/// Reviews an account's pending proposals against the signer set of its restored state. A +/// proposal none of those signers proposed or approved is removed, and a proposal that stays +/// keeps only their approvals. +pub(super) fn review(restored: &AppliedState, pending: &[ProposalRecord]) -> Vec { + pending + .iter() + .map(|record| ReviewedProposal { + commitment: record.commitment.clone(), + status: record.proposal.status.clone(), + change: change_for(&restored.cosigners, &record.proposal), + }) + .collect() +} + +fn change_for(signers: &[ChainCosigner], proposal: &DeltaObject) -> Option { + let DeltaStatus::Pending { + timestamp, + proposer_id, + cosigner_sigs, + } = &proposal.status + else { + return None; + }; + let approvals = approvals_of(signers, proposal, cosigner_sigs); + let proposed_by_signer = signers + .iter() + .any(|signer| signer.commitment.eq_ignore_ascii_case(proposer_id)); + if approvals.is_empty() && !proposed_by_signer { + return Some(ProposalChange::Remove); + } + (approvals.len() != cosigner_sigs.len()).then(|| { + ProposalChange::DropApprovals(DeltaStatus::Pending { + timestamp: timestamp.clone(), + proposer_id: proposer_id.clone(), + cosigner_sigs: approvals, + }) + }) +} + +fn approvals_of( + signers: &[ChainCosigner], + proposal: &DeltaObject, + signatures: &[CosignerSignature], +) -> Vec { + let Ok(summary) = proposal_tx_summary(&proposal.delta_payload) else { + return Vec::new(); + }; + signatures + .iter() + .filter(|approval| { + verify_proposal_signature(&summary, &approval.signature).is_ok_and(|signer| { + signers.iter().any(|known| { + known.commitment.eq_ignore_ascii_case(&signer) + && known.scheme == Some(approval.signature.scheme()) + }) + }) + }) + .cloned() + .collect() +} diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 8b29a5d0..405226ff 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -12,6 +12,7 @@ use std::path::PathBuf; #[cfg(feature = "postgres")] use server::builder::handle::StartupRecoveryError; +use server::builder::handle::StartupRecoveryMode; const STARTUP_RECOVERY_ENV: &str = "GUARDIAN_STARTUP_ACCOUNT_RECOVERY"; @@ -20,18 +21,20 @@ async fn main() { dotenvy::dotenv().ok(); let startup_recovery = match env::var(STARTUP_RECOVERY_ENV) { - Ok(value) => value.trim().parse::().unwrap_or_else(|_| { - eprintln!("{STARTUP_RECOVERY_ENV} must be true or false"); - std::process::exit(2); - }), - Err(env::VarError::NotPresent) => false, + Ok(value) => value + .parse::() + .unwrap_or_else(|error| { + eprintln!("{STARTUP_RECOVERY_ENV} {error}"); + std::process::exit(2); + }), + Err(env::VarError::NotPresent) => StartupRecoveryMode::Off, Err(env::VarError::NotUnicode(_)) => { eprintln!("{STARTUP_RECOVERY_ENV} must be valid UTF-8"); std::process::exit(2); } }; #[cfg(not(feature = "postgres"))] - if startup_recovery { + if startup_recovery != StartupRecoveryMode::Off { eprintln!("Account recovery requires a Postgres build"); std::process::exit(2); } @@ -106,8 +109,8 @@ async fn main() { .expect("Failed to build server"); #[cfg(feature = "postgres")] - if startup_recovery { - match server.recover_accounts_on_startup().await { + { + match server.recover_accounts_on_startup(startup_recovery).await { Ok(()) => {} Err(error @ StartupRecoveryError::Incomplete(_)) => { tracing::error!(%error, "Startup account recovery did not finish; serving without it"); diff --git a/crates/server/src/schema.rs b/crates/server/src/schema.rs index 3aee4820..3b701b2c 100644 --- a/crates/server/src/schema.rs +++ b/crates/server/src/schema.rs @@ -38,6 +38,7 @@ diesel::table! { account_id -> Varchar, prior_state -> Jsonb, prior_auth -> Jsonb, + prior_proposals -> Jsonb, #[max_length = 128] recovered_commitment -> Varchar, recovered_at -> Timestamptz, diff --git a/crates/server/src/storage/postgres_recovery.rs b/crates/server/src/storage/postgres_recovery.rs index 57a94892..cd3bc9e1 100644 --- a/crates/server/src/storage/postgres_recovery.rs +++ b/crates/server/src/storage/postgres_recovery.rs @@ -2,15 +2,16 @@ use std::{sync::Arc, time::Duration}; use chrono::{DateTime, Utc}; use diesel::prelude::*; -use diesel::sql_types::{BigInt, Bool, Jsonb, Nullable, Text, Timestamptz}; +use diesel::sql_types::{Array, BigInt, Bool, Jsonb, Nullable, Text, Timestamptz}; use diesel_async::scoped_futures::{ScopedBoxFuture, ScopedFutureExt}; use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl, SimpleAsyncConnection}; use serde_json::Value; use tokio::time::timeout; use crate::coordination::{LeaderElector, Lease, postgres::PgLeaseElector}; +use crate::delta_object::DeltaStatus; use crate::metadata::{Auth, MidenNetworkType, NetworkConfig}; -use crate::schema::{account_metadata, states}; +use crate::schema::{account_metadata, delta_proposals, states}; use crate::storage::LeaseFence; use crate::storage::encryption::{cipher::StorageCipher, envelope::RecordAad}; @@ -23,16 +24,6 @@ const UNSETTLED_ACCOUNT_WORK: &str = " SELECT 1 FROM deltas pending WHERE pending.account_id = st.account_id AND pending.status_kind IN ('candidate', 'retained') ) THEN 'a delta awaits its on-chain outcome' - WHEN EXISTS ( - SELECT 1 FROM deltas abandoned WHERE abandoned.account_id = st.account_id - AND abandoned.status_kind = 'discarded' - AND abandoned.status->>'reason' = 'client_abandoned' - ) THEN 'a client-abandoned delta is on record' - WHEN EXISTS (SELECT 1 FROM superseded_deltas d WHERE d.account_id = st.account_id) - THEN 'a superseded delta is retained' - WHEN EXISTS ( - SELECT 1 FROM delta_proposals p WHERE p.account_id = st.account_id AND p.status_kind = 'pending' - ) THEN 'a proposal is pending' WHEN EXISTS ( SELECT 1 FROM execution_reservations r WHERE r.account_id = st.account_id AND r.released_at IS NULL ) THEN 'an execution is reserved' @@ -103,6 +94,20 @@ impl PendingOutcome { } } +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ProposalChange { + Remove, + DropApprovals(DeltaStatus), +} + +/// A pending proposal as the scan read it, and what the restore does with it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ReviewedProposal { + pub commitment: String, + pub status: DeltaStatus, + pub change: Option, +} + #[derive(Debug, Default, Clone, PartialEq, Eq, diesel::QueryableByName)] pub(crate) struct ScanProgress { #[diesel(sql_type = Text)] @@ -370,10 +375,36 @@ impl RecoveryStore { candidate: &RecoveryCandidate, state_json: &Value, auth: &Auth, + proposals: &[ReviewedProposal], now: DateTime, lease: &Lease, ) -> Result { let account_id = &candidate.account_id; + let mut reviewed = proposals + .iter() + .map(|proposal| { + serde_json::to_value(&proposal.status) + .map(|status| (proposal.commitment.clone(), status)) + .map_err(|error| error.to_string()) + }) + .collect::, _>>()?; + reviewed.sort_by(|left, right| left.0.cmp(&right.0)); + let changes = proposals + .iter() + .filter_map(|proposal| { + let change = match proposal.change.as_ref()? { + ProposalChange::Remove => Ok(None), + ProposalChange::DropApprovals(status) => serde_json::to_value(status) + .map(Some) + .map_err(|error| error.to_string()), + }; + Some(change.map(|change| (proposal.commitment.clone(), change))) + }) + .collect::, _>>()?; + let changed: Vec = changes + .iter() + .map(|(commitment, _)| commitment.clone()) + .collect(); let canonical_nonce = candidate .canonical_nonce .ok_or("Recovery target has no canonical nonce")?; @@ -409,19 +440,45 @@ impl RecoveryStore { if !lease_fence_is_current(conn, &fence).await? { return Ok(false); } + if pending_proposals(conn, account_id).await? != reviewed { + return Ok(false); + } diesel::sql_query( "INSERT INTO account_recovery_backups - (account_id, prior_state, prior_auth, recovered_commitment) - SELECT st.account_id, to_jsonb(st), m.auth, $2 + (account_id, prior_state, prior_auth, prior_proposals, + recovered_commitment) + SELECT st.account_id, to_jsonb(st), m.auth, + COALESCE(( + SELECT jsonb_agg(to_jsonb(p) ORDER BY p.commitment) + FROM delta_proposals p + WHERE p.account_id = st.account_id AND p.commitment = ANY($3) + ), '[]'::jsonb), + $2 FROM states st JOIN account_metadata m USING (account_id) WHERE st.account_id = $1", ) .bind::(account_id) .bind::(canonical_commitment) + .bind::, _>(&changed) .execute(conn) .await?; + for (commitment, kept_status) in &changes { + let proposal = delta_proposals::table + .filter(delta_proposals::account_id.eq(account_id)) + .filter(delta_proposals::commitment.eq(commitment)); + match kept_status { + Some(status) => { + diesel::update(proposal) + .set(delta_proposals::status.eq(status)) + .execute(conn) + .await? + } + None => diesel::delete(proposal).execute(conn).await?, + }; + } + let written_nonce = diesel::update(states::table.find(account_id)) .set(( states::commitment.eq(canonical_commitment), @@ -481,6 +538,34 @@ impl RecoveryStore { } } +/// The account's pending proposals and their statuses, locked for the transaction. +async fn pending_proposals( + conn: &mut AsyncPgConnection, + account_id: &str, +) -> QueryResult> { + #[derive(diesel::QueryableByName)] + struct PendingProposal { + #[diesel(sql_type = Text)] + commitment: String, + #[diesel(sql_type = Jsonb)] + status: Value, + } + + let mut pending: Vec<(String, Value)> = diesel::sql_query( + "SELECT commitment, status FROM delta_proposals + WHERE account_id = $1 AND status_kind = 'pending' + FOR UPDATE", + ) + .bind::(account_id) + .load::(conn) + .await? + .into_iter() + .map(|proposal| (proposal.commitment, proposal.status)) + .collect(); + pending.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(pending) +} + fn miden_network_config(network_type: MidenNetworkType) -> Result { serde_json::to_value(NetworkConfig::Miden { network_type }).map_err(|error| error.to_string()) } diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index d7b98ae7..d1d71650 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -30,7 +30,7 @@ the runtime env vars in this document. | `GUARDIAN_KEYSTORE_PATH` | `/var/guardian/keystore` | any | Local Falcon/ECDSA key files (ACK signers and per-account creds). | | `GUARDIAN_DB_POOL_MAX_SIZE` | `16`; `32` when `GUARDIAN_ENV=prod` | `postgres` | Storage backend pool size. | | `GUARDIAN_METADATA_DB_POOL_MAX_SIZE` | matches storage | `postgres` | Metadata backend pool size; usually leave equal. | -| `GUARDIAN_STARTUP_ACCOUNT_RECOVERY` | `false` | `postgres` | Checks recovery candidates for up to 5 minutes before starting API listeners. Later startups continue where the scan stopped. Use with stop-before-start upgrades and one replica. Guardian skips recovery when `GUARDIAN_MAX_REPLICAS` exceeds `1`. If another process holds the recovery lease, or the server cannot confirm it holds it alone, the server exits before opening its listeners. AWS deployments set it through Terraform variable `guardian_startup_account_recovery`, which also lengthens the task health check timings. See [Startup account recovery](./PRODUCTION.md#startup-account-recovery). | +| `GUARDIAN_STARTUP_ACCOUNT_RECOVERY` | `off` | `postgres` | `off`, `dry-run` or `apply`. Checks account records against the Miden network for up to 5 minutes before starting API listeners. `dry-run` logs what it would change and writes nothing. `apply` continues on later startups where it stopped; use it with stop-before-start upgrades and one replica, and Guardian skips it when `GUARDIAN_MAX_REPLICAS` exceeds `1`. In `apply`, if another process holds the recovery lease, or the server cannot confirm it holds it alone, the server exits before opening its listeners. AWS deployments set it through Terraform variable `guardian_startup_account_recovery`, which also lengthens the task health check timings. See [Startup account recovery](./PRODUCTION.md#startup-account-recovery). | | `GUARDIAN_METADATA_CACHE_MAX_BYTES` | `268435456` (256 MiB) | `postgres` | Memory bound of the per-replica account metadata cache used by read routes. Every read still confirms the copy is current against the database (one indexed lookup of a version the database raises on every metadata update), so the cache never serves stale authorization data; it only saves loading and parsing the row. `0` disables caching. | | `GUARDIAN_CANONICALIZATION_FAST_PROMOTION_ENABLED` | `true` | any | Enables the additional promotion-only pass for recent candidates. Set to `false` to use only the full canonicalization interval; AWS deployments can set Terraform variable `guardian_canonicalization_fast_promotion_enabled = false`. | | `GUARDIAN_CANONICALIZATION_MAX_CONCURRENT_ACCOUNTS` | `10`; `50` when `GUARDIAN_ENV=prod` | any | Accounts one canonicalization pass processes in parallel; `1` = fully sequential. Each account holds a DB connection only during its short fenced transactions (the dominant cost is a connectionless chain RPC), so this may exceed `GUARDIAN_DB_POOL_MAX_SIZE`; simultaneous write bursts just queue briefly at the pool. | diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index 8347f3f1..c6172702 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -354,23 +354,34 @@ The verification and restore procedure is in Guardian can check its stored Miden account records against the Miden network when it starts, and bring a record back in line with the account's canonical -history when the two disagree. The check is off by default. +history when the two disagree. `GUARDIAN_STARTUP_ACCOUNT_RECOVERY` selects +the mode: `off` (the default), `dry-run` or `apply`. It needs a Guardian +backed by Postgres. With the bundled Compose files, set it in `.env`. -**Enabling it** +**Previewing with a dry run** -- Set `GUARDIAN_STARTUP_ACCOUNT_RECOVERY=true` on a single Guardian instance - backed by Postgres. With the bundled Compose files, set it in `.env`. +Start Guardian once with `GUARDIAN_STARTUP_ACCOUNT_RECOVERY=dry-run` first. A +dry run performs the same checks, logs what `apply` would do with each +account, and writes nothing: no account record, no report row and no +progress. It takes no lease, so it runs with any number of replicas, and it +checks from the first account each time. The final log line counts the +accounts that would be restored (`restorable`), deferred and left unresolved. + +**Applying** + +- Set `GUARDIAN_STARTUP_ACCOUNT_RECOVERY=apply` on a single Guardian instance. - Stop the old Guardian process before the new one starts. Rolling deployments overlap old and new servers even with a target of one instance, - so keep the flag off for those. -- Guardian skips the check when `GUARDIAN_MAX_REPLICAS` exceeds `1`. + so do not use `apply` for those. +- Guardian skips `apply` when `GUARDIAN_MAX_REPLICAS` exceeds `1`. - On the AWS stack in `infra/`, set Terraform variable - `guardian_startup_account_recovery = true` for that one deploy. It passes - the flag to the task and lengthens the container health check start period - and the load balancer grace period, so the task is not replaced while its - listeners are closed. Running a single task (desired count and autoscaling - maximum of one) and stopping the running task before the deploy remain - manual steps. Set the variable back to `false` once the check has completed. + `guardian_startup_account_recovery` to `"dry-run"` or `"apply"` for that one + deploy. It passes the mode to the task and lengthens the container health + check start period and the load balancer grace period, so the task is not + replaced while its listeners are closed. For `apply`, running a single task + (desired count and autoscaling maximum of one) and stopping the running task + before the deploy remain manual steps. Set the variable back to `"off"` + once the check has completed. **What it does** @@ -388,9 +399,15 @@ Miden node for the account's current commitment. - Otherwise the account is left unchanged and listed in `account_recovery_pending`. -A paused account, or one with work in progress such as a delta awaiting its -outcome, a pending proposal or a reserved execution, is deferred and checked -again on a later startup. +A restore also reviews the account's pending proposals against the signers of +the restored state. A proposal that one of those signers proposed or approved +stays, keeping only their approvals. Any other pending proposal is removed. +Every proposal a restore changes or removes is saved with the previous record +in `account_recovery_backups.prior_proposals`. + +A paused account, or one with work in flight (a delta awaiting its on-chain +outcome or a reserved execution), is deferred and checked again on a later +startup. **How long it takes** @@ -400,29 +417,30 @@ again on a later startup. writes. Allow for that in any startup health check or grace period. - The check stops for this startup after three account checks in a row fail, for example while the Miden node is unreachable. -- Guardian records where it stopped in - `account_recovery_startup.resume_after`, and the next startup with the flag - on continues from there. +- In `apply`, Guardian records where it stopped in + `account_recovery_startup.resume_after`, and the next startup in `apply` + continues from there. - The check completes when one startup covers every account, from the first to the last, without deferring any. Guardian then records how many accounts it could not bring up to date in `account_recovery_startup.unresolved_accounts`, and later startups skip the - check. Until then, each startup with the flag on runs it again. + check. Until then, each startup in `apply` runs it again. - An account Guardian holds canonical deltas for, but the configured node has no record of, is deferred. It keeps the check open until the node knows the account or the account is released. **If it cannot run** -Guardian exits before opening its listeners if another process holds the -recovery lease, or if it cannot confirm it holds the lease alone. It first -waits up to 35 seconds for a lease a stopped process left behind. Any other -error is logged, and Guardian opens its listeners without finishing the check. +In `apply`, Guardian exits before opening its listeners if another process +holds the recovery lease, or if it cannot confirm it holds the lease alone. It +first waits up to 35 seconds for a lease a stopped process left behind. Any +other error, and any error in a dry run, is logged, and Guardian opens its +listeners without finishing the check. **Reviewing the result** -`account_recovery_pending` has one row for each account that was checked and -not brought up to date. `outcome` is `deferred` (a later startup checks it +After `apply`, `account_recovery_pending` has one row for each account that +was checked and not brought up to date. `outcome` is `deferred` (a later startup checks it again) or `unresolved` (Guardian does not hold what it needs), and `reason` says why. When the canonical history held here does not reach back to the account's creation, `history_starts_at_nonce` and `history_base_commitment` diff --git a/infra/README.md b/infra/README.md index 0d57ad80..3f267405 100644 --- a/infra/README.md +++ b/infra/README.md @@ -116,7 +116,7 @@ server_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/guardian-server # guardian_canonicalization_fast_promotion_enabled = false # guardian_max_pending_candidates_per_account = 4 # opt in to queueing chained candidates (server default 1, max 16; upgrade TS clients first, see docs/CONFIGURATION.md) # guardian_release_sweep_enabled = false # turns the release sweep's walk of the fleet off -# guardian_startup_account_recovery = true # one deploy only: single task, older task stopped first; see docs/PRODUCTION.md +# guardian_startup_account_recovery = "dry-run" # then "apply" for one deploy: single task, older task stopped first; see docs/PRODUCTION.md # guardian_block_watch_enabled = false # kill switch for the committed-block feed # Optional: dashboard operator Falcon public keys managed by Terraform @@ -304,7 +304,7 @@ aws ecr delete-repository --repository-name "$ECR_REPO_NAME" --force --region "$ | `guardian_canonicalization_fast_promotion_enabled` | `true` | Enables the recent-candidate promotion-only pass in the ECS task definition | | `guardian_max_pending_candidates_per_account` | server default (`1`) | `GUARDIAN_MAX_PENDING_CANDIDATES_PER_ACCOUNT`: chained candidate deltas one account may hold in flight (issue #17). `1` keeps one in-flight candidate per account; `2` to `16` opts in to queueing back-to-back transactions, after TypeScript clients are upgraded (see [`docs/CONFIGURATION.md`](../docs/CONFIGURATION.md)) | | `guardian_release_sweep_enabled` | `true` | Runs the release sweep's walk of the fleet (the rotation); `false` turns the walk off, and block watch too unless `guardian_block_watch_enabled` is set | -| `guardian_startup_account_recovery` | `false` | `GUARDIAN_STARTUP_ACCOUNT_RECOVERY`: runs account recovery before the listeners open, and lengthens the container health check start period (to 300 s) and the load balancer grace period (to 420 s) so the task is not replaced during the scan. Needs a single task and a deploy with no older task still running; turn it off again once the scan has completed. See [Startup account recovery](../docs/PRODUCTION.md#startup-account-recovery) | +| `guardian_startup_account_recovery` | `off` | `GUARDIAN_STARTUP_ACCOUNT_RECOVERY`: `off`, `dry-run` or `apply`. Checks account records before the listeners open; `dry-run` reports what it would change and writes nothing. Any value but `off` lengthens the container health check start period (to 300 s) and the load balancer grace period (to 420 s) so the task is not replaced during the check. `apply` needs a single task and a deploy with no older task still running; set it back to `off` once the check has completed. See [Startup account recovery](../docs/PRODUCTION.md#startup-account-recovery) | | `guardian_release_sweep_rotation_seconds` | server default (`21600`) | Target time for one release sweep walk of the fleet; `604800` (weekly) is enough once block watch is healthy | | `guardian_block_watch_enabled` | unset (follows `guardian_release_sweep_enabled`) | Runs block watch (issue #585): switch detection from the node's committed-block feed; `false` is its kill switch, `true` runs it even with the release sweep's walk turned off | | `guardian_block_watch_transport` | `auto` | `GUARDIAN_BLOCK_WATCH_TRANSPORT`: `auto` (the block stream, polling when it is unavailable), `stream`, or `poll` | diff --git a/infra/data.tf b/infra/data.tf index da3fe7c4..3ba84c87 100644 --- a/infra/data.tf +++ b/infra/data.tf @@ -171,8 +171,8 @@ locals { # task left behind, and the writes in progress when the budget ends. 300 is # the longest container start period ECS accepts; the health check's three # retries at 30 seconds cover the remainder. - server_health_check_start_period_seconds = var.guardian_startup_account_recovery ? 300 : 10 - server_health_check_grace_period_seconds = var.guardian_startup_account_recovery ? 420 : 30 + server_health_check_start_period_seconds = var.guardian_startup_account_recovery == "off" ? 10 : 300 + server_health_check_grace_period_seconds = var.guardian_startup_account_recovery == "off" ? 30 : 420 effective_server_desired_count = var.server_desired_count != null ? var.server_desired_count : (local.is_prod ? 2 : 1) effective_server_autoscaling_enabled = var.server_autoscaling_enabled != null ? var.server_autoscaling_enabled : local.is_prod diff --git a/infra/ecs.tf b/infra/ecs.tf index 3225248e..d02cdb74 100644 --- a/infra/ecs.tf +++ b/infra/ecs.tf @@ -204,7 +204,7 @@ resource "aws_ecs_task_definition" "server" { }, { name = "GUARDIAN_STARTUP_ACCOUNT_RECOVERY" - value = tostring(var.guardian_startup_account_recovery) + value = var.guardian_startup_account_recovery }, { name = "GUARDIAN_BLOCK_WATCH_TRANSPORT" diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 975c6c47..e2a2fad2 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -130,7 +130,7 @@ server_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/guardian-server # guardian_canonicalization_fast_promotion_enabled = false # guardian_max_pending_candidates_per_account = 4 # opt in to queueing chained candidates per account (server default 1, max 16) # guardian_release_sweep_enabled = false # turns the release sweep's walk of the fleet off -# guardian_startup_account_recovery = true # one deploy only: single task, older task stopped first; see docs/PRODUCTION.md +# guardian_startup_account_recovery = "dry-run" # then "apply" for one deploy: single task, older task stopped first; see docs/PRODUCTION.md # guardian_release_sweep_rotation_seconds = 21600 # 604800 (weekly) once block watch is healthy # guardian_release_sweep_max_rate_per_second = 5 # guardian_block_watch_enabled = false # kill switch for the committed-block feed (issue #585); unset follows guardian_release_sweep_enabled diff --git a/infra/variables.tf b/infra/variables.tf index b331c2ea..abe795c4 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -572,17 +572,23 @@ variable "guardian_release_sweep_enabled" { variable "guardian_startup_account_recovery" { description = <<-EOT - Whether the server runs account recovery before it opens its listeners - (GUARDIAN_STARTUP_ACCOUNT_RECOVERY). true also lengthens the container - health check start period and the load balancer grace period, so a task - is not replaced while the scan keeps the listeners closed. The operator - arranges the rest: the server skips recovery unless it runs as a single - task, and the deploy must not overlap an older task, so stop the running - task before the deploy that turns this on. Turn it off again once the - scan has completed. + Whether the server checks its account records before it opens its + listeners (GUARDIAN_STARTUP_ACCOUNT_RECOVERY): off, dry-run (reports what + it would change and writes nothing) or apply. Any value but off also + lengthens the container health check start period and the load balancer + grace period, so a task is not replaced while the check keeps the + listeners closed. For apply the operator arranges the rest: the server + skips the check unless it runs as a single task, and the deploy must not + overlap an older task, so stop the running task before the deploy that + turns this on. Set it back to off once the check has completed. EOT - type = bool - default = false + type = string + default = "off" + + validation { + condition = contains(["off", "dry-run", "apply"], var.guardian_startup_account_recovery) + error_message = "guardian_startup_account_recovery must be off, dry-run or apply." + } } variable "guardian_block_watch_enabled" { From 4cc030ad17d81678e702356a99de9a4ec2e6f330 Mon Sep 17 00:00:00 2001 From: Zeljko Date: Thu, 19 Nov 2026 17:21:34 +0100 Subject: [PATCH 3/8] fix(server): trim proposal payloads and tighten the recovery proposal review - A proposal that keeps only some approvals now has them dropped from its payload as well as its status, so both client SDKs read the same approvals. The payload is encrypted the way the storage layer stores it. - The check that an account's proposals are unchanged since the review compares typed statuses, as the storage layer reads them. - Proposals that cannot be read no longer leave the account unresolved: the restore proceeds and leaves them as they are. - A dry run names each proposal a restore would change and says when apply would not run (more than one replica, no canonicalization, or the check already completed). - Tests cover the unchanged-proposals check, a partial trim with an ECDSA approval through encrypted storage, a kept proposal staying out of the backup, a client-abandoned delta no longer deferring, and a dry run writing nothing. The mode parsing test moves out of the Postgres-only module. --- crates/server/src/builder/handle.rs | 63 +++- crates/server/src/jobs/account_recovery.rs | 72 ++-- .../jobs/account_recovery/postgres_tests.rs | 324 ++++++++++++------ .../src/jobs/account_recovery/proposals.rs | 58 +++- .../server/src/storage/postgres_recovery.rs | 142 ++++++-- docs/PRODUCTION.md | 13 +- 6 files changed, 475 insertions(+), 197 deletions(-) diff --git a/crates/server/src/builder/handle.rs b/crates/server/src/builder/handle.rs index bf9e8b5f..dadd7888 100644 --- a/crates/server/src/builder/handle.rs +++ b/crates/server/src/builder/handle.rs @@ -116,31 +116,41 @@ impl ServerHandle { &self, mode: StartupRecoveryMode, ) -> Result<(), StartupRecoveryError> { - let dry_run = match mode { - StartupRecoveryMode::Off => return Ok(()), - StartupRecoveryMode::DryRun => true, - StartupRecoveryMode::Apply => false, - }; + if mode == StartupRecoveryMode::Off { + return Ok(()); + } let (store, network_type) = self .account_recovery .as_ref() .ok_or_else(|| "Account recovery requires shared Postgres storage".to_owned())?; - if dry_run { - return crate::jobs::account_recovery::dry_run(&self.app_state, store, *network_type) - .await; - } - if crate::middleware::rate_limit::max_replicas_from_env()? != 1 { - tracing::warn!( - "Skipping startup account recovery because GUARDIAN_MAX_REPLICAS exceeds 1" - ); - return Ok(()); - } - if self.app_state.canonicalization.is_none() { - return Err(StartupRecoveryError::Incomplete( + let single_replica = crate::middleware::rate_limit::max_replicas_from_env()? == 1; + let canonicalizes = self.app_state.canonicalization.is_some(); + match mode { + StartupRecoveryMode::Off => Ok(()), + StartupRecoveryMode::DryRun => { + if !single_replica { + tracing::warn!( + "Dry run: apply would be skipped because GUARDIAN_MAX_REPLICAS exceeds 1" + ); + } + if !canonicalizes { + tracing::warn!("Dry run: apply would not run without canonicalization"); + } + crate::jobs::account_recovery::dry_run(&self.app_state, store, *network_type).await + } + StartupRecoveryMode::Apply if !single_replica => { + tracing::warn!( + "Skipping startup account recovery because GUARDIAN_MAX_REPLICAS exceeds 1" + ); + Ok(()) + } + StartupRecoveryMode::Apply if !canonicalizes => Err(StartupRecoveryError::Incomplete( "Account recovery requires canonicalization".into(), - )); + )), + StartupRecoveryMode::Apply => { + crate::jobs::account_recovery::run_once(&self.app_state, store, *network_type).await + } } - crate::jobs::account_recovery::run_once(&self.app_state, store, *network_type).await } /// Run the server with the configured settings @@ -898,3 +908,18 @@ fn start_session_sweep_worker(state: AppState) { } }); } + +#[cfg(test)] +mod startup_recovery_mode_tests { + use super::StartupRecoveryMode; + + #[test] + fn the_mode_has_three_values_and_takes_no_boolean() { + assert_eq!("off".parse(), Ok(StartupRecoveryMode::Off)); + assert_eq!(" dry-run ".parse(), Ok(StartupRecoveryMode::DryRun)); + assert_eq!("apply".parse(), Ok(StartupRecoveryMode::Apply)); + for rejected in ["true", "false", "", "Apply"] { + assert!(rejected.parse::().is_err()); + } + } +} diff --git a/crates/server/src/jobs/account_recovery.rs b/crates/server/src/jobs/account_recovery.rs index bd876b18..d677ddc3 100644 --- a/crates/server/src/jobs/account_recovery.rs +++ b/crates/server/src/jobs/account_recovery.rs @@ -15,8 +15,8 @@ use crate::network::{ use crate::state::AppState; use crate::storage::is_record_corruption; use crate::storage::postgres::recovery::{ - PendingOutcome, ProposalChange, RecoveryCandidate, RecoveryStore, ReviewedProposal, - ScanProgress, + PendingOutcome, ProposalChange, ProposalReview, RecoveryCandidate, RecoveryStore, + ReviewedProposal, ScanProgress, }; const PAGE_SIZE: i64 = 100; @@ -42,23 +42,38 @@ enum RecoveryOutcome { struct Restoration { state: AppliedState, auth: Auth, - proposals: Vec, + proposals: ProposalReview, } impl Restoration { - fn proposals_removed(&self) -> usize { - self.proposals_changed(|change| matches!(change, ProposalChange::Remove)) - } - - fn proposals_trimmed(&self) -> usize { - self.proposals_changed(|change| matches!(change, ProposalChange::DropApprovals(_))) + /// The proposals a restore removes or rewrites. + fn changed_proposals(&self) -> impl Iterator { + let reviewed = match &self.proposals { + ProposalReview::Reviewed(reviewed) => reviewed.as_slice(), + ProposalReview::Unreadable => &[], + }; + reviewed + .iter() + .filter(|proposal| proposal.change != ProposalChange::Keep) } - fn proposals_changed(&self, matches: impl Fn(&ProposalChange) -> bool) -> usize { - self.proposals - .iter() - .filter(|proposal| proposal.change.as_ref().is_some_and(&matches)) - .count() + fn log_proposal_changes(&self, account_id: &str, dry_run: bool) { + if self.proposals == ProposalReview::Unreadable { + tracing::warn!( + %account_id, + dry_run, + "Account recovery could not read the pending proposals and leaves them as they are" + ); + } + for proposal in self.changed_proposals() { + tracing::info!( + %account_id, + proposal = %proposal.commitment, + change = proposal.change.label(), + dry_run, + "Account recovery proposal review" + ); + } } } @@ -161,6 +176,18 @@ pub(crate) async fn dry_run( store: &RecoveryStore, network_type: MidenNetworkType, ) -> Result<(), StartupRecoveryError> { + if bounded_db_call( + "read the recovery checkpoint", + store.startup_completed(network_type), + ) + .await + .unwrap_or(false) + { + tracing::info!( + ?network_type, + "Dry run: account recovery already completed, so apply would skip the scan" + ); + } run( state, store, @@ -328,10 +355,10 @@ async fn run( summary.restorable += 1; tracing::info!( account_id = %candidate.account_id, - proposals_removed = restoration.proposals_removed(), - proposals_trimmed = restoration.proposals_trimmed(), + proposals_changed = restoration.changed_proposals().count(), "Dry run: the account would be restored" ); + restoration.log_proposal_changes(&candidate.account_id, true); PendingRecord::Clear } ScanWrites::Apply(lease) => { @@ -350,10 +377,10 @@ async fn run( summary.restored += 1; tracing::info!( account_id = %candidate.account_id, - proposals_removed = restoration.proposals_removed(), - proposals_trimmed = restoration.proposals_trimmed(), + proposals_changed = restoration.changed_proposals().count(), "Restored chain-verified account state and signers" ); + restoration.log_proposal_changes(&candidate.account_id, false); PendingRecord::Clear } Ok(false) => { @@ -602,18 +629,15 @@ async fn check_candidate( } } - let pending = match state + let proposals = match state .storage .pull_pending_proposals(&candidate.account_id) .await { - Ok(pending) => pending, - Err(reason) if is_record_corruption(&reason) => { - return Ok(RecoveryOutcome::Unresolved(reason)); - } + Ok(pending) => ProposalReview::Reviewed(proposals::review(&applied, &pending)), + Err(reason) if is_record_corruption(&reason) => ProposalReview::Unreadable, Err(reason) => return Err(reason), }; - let proposals = proposals::review(&applied, &pending); Ok(RecoveryOutcome::Verified(Box::new(Restoration { state: applied, auth, diff --git a/crates/server/src/jobs/account_recovery/postgres_tests.rs b/crates/server/src/jobs/account_recovery/postgres_tests.rs index f36fefec..09ea9c95 100644 --- a/crates/server/src/jobs/account_recovery/postgres_tests.rs +++ b/crates/server/src/jobs/account_recovery/postgres_tests.rs @@ -200,8 +200,8 @@ fn a_proposal_review_keeps_what_the_restored_signers_proposed_or_approved() { let account_id = "0x7b7b7b7a7b7b7b017b7b7b7b7b7b7b"; let owner = TestSigner::new(); let other_key = TestSigner::new(); - let payload = serde_json::json!({ "tx_summary": create_test_delta_payload(account_id) }); - let message = crate::services::proposal_signature::proposal_tx_summary(&payload) + let tx_summary = create_test_delta_payload(account_id); + let message = TransactionSummary::from_json(&tx_summary) .unwrap() .to_commitment(); let approval = |signer: &TestSigner| CosignerSignature { @@ -211,21 +211,34 @@ fn a_proposal_review_keeps_what_the_restored_signers_proposed_or_approved() { timestamp: "2026-11-14T12:50:00Z".into(), signer_id: signer.commitment_hex.clone(), }; - let pending = |proposer: &TestSigner, approvals: Vec| DeltaStatus::Pending { + let payload = |approvals: &[CosignerSignature]| { + let signatures: Vec = approvals + .iter() + .map(|approval| { + serde_json::json!({ + "signer_id": approval.signer_id, + "signature": approval.signature, + }) + }) + .collect(); + serde_json::json!({ "tx_summary": tx_summary, "signatures": signatures }) + }; + let pending = |proposer: &TestSigner, approvals: &[CosignerSignature]| DeltaStatus::Pending { timestamp: "2026-11-14T12:45:00Z".into(), proposer_id: proposer.commitment_hex.clone(), - cosigner_sigs: approvals, + cosigner_sigs: approvals.to_vec(), }; - let record = |commitment: &str, status: DeltaStatus| ProposalRecord { - account_id: account_id.into(), - commitment: commitment.into(), - proposal: DeltaObject { + let record = + |commitment: &str, proposer: &TestSigner, approvals: &[CosignerSignature]| ProposalRecord { account_id: account_id.into(), - delta_payload: payload.clone(), - status, - ..Default::default() - }, - }; + commitment: commitment.into(), + proposal: DeltaObject { + account_id: account_id.into(), + delta_payload: payload(approvals), + status: pending(proposer, approvals), + ..Default::default() + }, + }; let restored = |scheme: SignatureScheme| AppliedState { state_json: serde_json::Value::Null, commitment: "0xrestored".into(), @@ -236,18 +249,17 @@ fn a_proposal_review_keeps_what_the_restored_signers_proposed_or_approved() { }], guardian_commitment: None, }; + let own = approval(&owner); + let other = approval(&other_key); let records = [ - record("signed", pending(&owner, vec![approval(&owner)])), - record( - "mixed", - pending(&owner, vec![approval(&owner), approval(&other_key)]), - ), - record("approved", pending(&other_key, vec![approval(&owner)])), - record("unrelated", pending(&other_key, vec![approval(&other_key)])), - record("unsigned", pending(&owner, Vec::new())), + record("signed", &owner, std::slice::from_ref(&own)), + record("mixed", &owner, &[own.clone(), other.clone()]), + record("approved", &other_key, std::slice::from_ref(&own)), + record("unrelated", &other_key, std::slice::from_ref(&other)), + record("unsigned", &owner, &[]), ]; - let changes: Vec> = + let changes: Vec = proposals::review(&restored(SignatureScheme::Falcon), &records) .into_iter() .map(|reviewed| reviewed.change) @@ -255,34 +267,27 @@ fn a_proposal_review_keeps_what_the_restored_signers_proposed_or_approved() { assert_eq!( changes, [ - None, - Some(ProposalChange::DropApprovals(pending( - &owner, - vec![approval(&owner)] - ))), - None, - Some(ProposalChange::Remove), - None, + ProposalChange::Keep, + ProposalChange::DropApprovals { + status: pending(&owner, std::slice::from_ref(&own)), + delta_payload: payload(std::slice::from_ref(&own)), + }, + ProposalChange::Keep, + ProposalChange::Remove, + ProposalChange::Keep, ] ); let under_another_scheme = proposals::review(&restored(SignatureScheme::Ecdsa), &records[..1]); assert_eq!( under_another_scheme[0].change, - Some(ProposalChange::DropApprovals(pending(&owner, Vec::new()))) + ProposalChange::DropApprovals { + status: pending(&owner, &[]), + delta_payload: payload(&[]), + } ); } -#[test] -fn the_startup_recovery_mode_has_three_values() { - use crate::builder::handle::StartupRecoveryMode; - - assert_eq!("off".parse(), Ok(StartupRecoveryMode::Off)); - assert_eq!(" dry-run ".parse(), Ok(StartupRecoveryMode::DryRun)); - assert_eq!("apply".parse(), Ok(StartupRecoveryMode::Apply)); - assert!("true".parse::().is_err()); -} - #[tokio::test] async fn partial_history_fixture_cannot_rebuild_a_missing_base() { let network = MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal); @@ -539,7 +544,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e &candidate, &canonical.state_json, &expected_auth, - &[], + &ProposalReview::Reviewed(Vec::new()), state.clock.now(), &lease ) @@ -577,10 +582,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e .update_delta_status( account_id, 3, - DeltaStatus::Discarded { - timestamp: "2026-11-14T12:40:00Z".into(), - reason: None, - }, + DeltaStatus::discarded_client_abandoned("2026-11-14T12:40:00Z".into()), ) .await .unwrap(); @@ -657,62 +659,115 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e drop(conn); let other_key = crate::testing::helpers::TestSigner::new(); - let proposal_payload = serde_json::json!({ - "tx_summary": crate::testing::helpers::create_test_delta_payload(account_id), - }); - let other_approval = crate::delta_object::CosignerSignature { - signature: guardian_shared::ProposalSignature::Falcon { - signature: other_key.sign_word( - crate::services::proposal_signature::proposal_tx_summary(&proposal_payload) - .unwrap() - .to_commitment(), - ), - }, - timestamp: "2026-11-14T12:50:00Z".into(), - signer_id: other_key.commitment_hex.clone(), + let tx_summary = crate::testing::helpers::create_test_delta_payload(account_id); + let message = TransactionSummary::from_json(&tx_summary) + .unwrap() + .to_commitment(); + let approval = |signature: guardian_shared::ProposalSignature, signer_id: &str| { + crate::delta_object::CosignerSignature { + signature, + timestamp: "2026-11-14T12:50:00Z".into(), + signer_id: signer_id.into(), + } }; - let pending_proposal = |nonce: u64, proposer: &str| DeltaObject { - account_id: account_id.clone(), - nonce, - prev_commitment: prior.commitment.clone(), - delta_payload: proposal_payload.clone(), - status: DeltaStatus::Pending { - timestamp: "2026-11-14T12:45:00Z".into(), - proposer_id: proposer.into(), - cosigner_sigs: vec![other_approval.clone()], + let other_approval = approval( + guardian_shared::ProposalSignature::Falcon { + signature: other_key.sign_word(message), }, - ..Default::default() - }; - storage - .submit_delta_proposal( + &other_key.commitment_hex, + ); + let signer_approval = approval( + guardian_shared::ProposalSignature::Ecdsa { + signature: signer.sign_word(message), + public_key: Some(signer.pubkey_hex.clone()), + message_format: guardian_shared::EcdsaMessageFormat::Raw, + }, + &signer.commitment_hex, + ); + let pending_proposal = + |nonce: u64, proposer: &str, approvals: &[crate::delta_object::CosignerSignature]| { + let signatures: Vec = approvals + .iter() + .map(|approval| { + serde_json::json!({ + "signer_id": approval.signer_id, + "signature": approval.signature, + }) + }) + .collect(); + DeltaObject { + account_id: account_id.clone(), + nonce, + prev_commitment: prior.commitment.clone(), + delta_payload: serde_json::json!({ + "tx_summary": tx_summary, + "signatures": signatures, + }), + status: DeltaStatus::Pending { + timestamp: "2026-11-14T12:45:00Z".into(), + proposer_id: proposer.into(), + cosigner_sigs: approvals.to_vec(), + }, + ..Default::default() + } + }; + let untouched = pending_proposal( + 6, + &signer.commitment_hex, + std::slice::from_ref(&signer_approval), + ); + for (commitment, proposal) in [ + ( "0xproposed-by-another-key", - &pending_proposal(4, &other_key.commitment_hex), - ) - .await - .unwrap(); - storage - .submit_delta_proposal( + pending_proposal( + 4, + &other_key.commitment_hex, + std::slice::from_ref(&other_approval), + ), + ), + ( "0xproposed-by-the-signer", - &pending_proposal(5, &signer.commitment_hex), + pending_proposal( + 5, + &signer.commitment_hex, + &[signer_approval.clone(), other_approval.clone()], + ), + ), + ("0xsigned-by-the-signer-only", untouched.clone()), + ] { + storage + .submit_delta_proposal(commitment, &proposal) + .await + .unwrap(); + } + + #[derive(diesel::QueryableByName, Debug, PartialEq, Eq)] + struct RecoveryRows { + #[diesel(sql_type = BigInt)] + pending: i64, + #[diesel(sql_type = BigInt)] + backups: i64, + #[diesel(sql_type = Text)] + progress: String, + } + let recovery_rows = || async { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "SELECT (SELECT count(*) FROM account_recovery_pending) AS pending, + (SELECT count(*) FROM account_recovery_backups) AS backups, + (SELECT COALESCE(string_agg(to_jsonb(s)::text, ',' ORDER BY s.network_config::text), '') + FROM account_recovery_startup s) AS progress", ) + .get_result::(&mut conn) + .await + .unwrap() + }; + let before_dry_run = recovery_rows().await; + assert_eq!((before_dry_run.pending, before_dry_run.backups), (1, 0)); + dry_run(&state, &store, MidenNetworkType::Testnet) .await .unwrap(); - - let preview = run( - &state, - &store, - MidenNetworkType::Testnet, - ScanWrites::DryRun, - &CancellationToken::new(), - "", - Instant::now() + Duration::from_secs(60), - ) - .await - .unwrap(); - assert_eq!( - (preview.checked, preview.restorable, preview.restored), - (1, 1, 0) - ); + assert_eq!(recovery_rows().await, before_dry_run); assert_eq!( storage.pull_state(account_id).await.unwrap().commitment, prior.commitment @@ -723,8 +778,56 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e .await .unwrap() .len(), - 2 + 3 + ); + + let reviewing = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + let reviewed_candidate = store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap() + .pop() + .unwrap(); + let RecoveryOutcome::Verified(reviewed) = + check_candidate(&state, &reviewed_candidate).await.unwrap() + else { + panic!("the account is restorable"); + }; + assert_eq!(reviewed.changed_proposals().count(), 2); + storage + .submit_delta_proposal( + "0xadded-after-the-review", + &pending_proposal(7, &signer.commitment_hex, &[]), + ) + .await + .unwrap(); + assert!( + !store + .apply( + &reviewed_candidate, + &reviewed.state.state_json, + &reviewed.auth, + &reviewed.proposals, + state.clock.now(), + &reviewing, + ) + .await + .unwrap() + ); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment ); + storage + .delete_delta_proposal(account_id, "0xadded-after-the-review") + .await + .unwrap(); + store.leader.release(reviewing).await.unwrap(); run_once(&state, &store, MidenNetworkType::Testnet) .await @@ -747,12 +850,25 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e ); let kept = storage.pull_pending_proposals(account_id).await.unwrap(); - assert_eq!(kept.len(), 1); - assert_eq!(kept[0].commitment, "0xproposed-by-the-signer"); - assert!(matches!( - &kept[0].proposal.status, - DeltaStatus::Pending { cosigner_sigs, .. } if cosigner_sigs.is_empty() - )); + assert_eq!( + kept.iter() + .map(|record| record.commitment.as_str()) + .collect::>(), + ["0xproposed-by-the-signer", "0xsigned-by-the-signer-only"] + ); + let trimmed = pending_proposal( + 5, + &signer.commitment_hex, + std::slice::from_ref(&signer_approval), + ); + assert_eq!( + (&kept[0].proposal.status, &kept[0].proposal.delta_payload), + (&trimmed.status, &trimmed.delta_payload) + ); + assert_eq!( + (&kept[1].proposal.status, &kept[1].proposal.delta_payload), + (&untouched.status, &untouched.delta_payload) + ); let held = contender .leader @@ -782,7 +898,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e &candidate, &canonical.state_json, &expected_auth, - &[], + &ProposalReview::Reviewed(Vec::new()), state.clock.now(), &lease ) @@ -1305,7 +1421,7 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen &Auth::MidenEcdsa { cosigner_commitments: vec![Word::from([1, 2, 3, 4_u32]).into_hex()], }, - &[], + &ProposalReview::Reviewed(Vec::new()), chrono::Utc::now(), &lease, ) diff --git a/crates/server/src/jobs/account_recovery/proposals.rs b/crates/server/src/jobs/account_recovery/proposals.rs index d876e5fc..f980bfa5 100644 --- a/crates/server/src/jobs/account_recovery/proposals.rs +++ b/crates/server/src/jobs/account_recovery/proposals.rs @@ -19,29 +19,36 @@ pub(super) fn review(restored: &AppliedState, pending: &[ProposalRecord]) -> Vec .collect() } -fn change_for(signers: &[ChainCosigner], proposal: &DeltaObject) -> Option { - let DeltaStatus::Pending { - timestamp, - proposer_id, - cosigner_sigs, - } = &proposal.status - else { - return None; +fn change_for(signers: &[ChainCosigner], proposal: &DeltaObject) -> ProposalChange { + let (timestamp, proposer_id, cosigner_sigs) = match &proposal.status { + DeltaStatus::Pending { + timestamp, + proposer_id, + cosigner_sigs, + } => (timestamp, proposer_id, cosigner_sigs), + DeltaStatus::Candidate { .. } + | DeltaStatus::Canonical { .. } + | DeltaStatus::Retained { .. } + | DeltaStatus::Discarded { .. } => return ProposalChange::Keep, }; let approvals = approvals_of(signers, proposal, cosigner_sigs); let proposed_by_signer = signers .iter() .any(|signer| signer.commitment.eq_ignore_ascii_case(proposer_id)); if approvals.is_empty() && !proposed_by_signer { - return Some(ProposalChange::Remove); + return ProposalChange::Remove; } - (approvals.len() != cosigner_sigs.len()).then(|| { - ProposalChange::DropApprovals(DeltaStatus::Pending { + if approvals.len() == cosigner_sigs.len() { + return ProposalChange::Keep; + } + ProposalChange::DropApprovals { + delta_payload: payload_with_approvals(&proposal.delta_payload, &approvals), + status: DeltaStatus::Pending { timestamp: timestamp.clone(), proposer_id: proposer_id.clone(), cosigner_sigs: approvals, - }) - }) + }, + } } fn approvals_of( @@ -65,3 +72,28 @@ fn approvals_of( .cloned() .collect() } + +/// The payload with its `signatures` list reduced to the signers of `approvals`. A proposal +/// stores each approval in its status and in its payload, and clients read either. +fn payload_with_approvals( + delta_payload: &serde_json::Value, + approvals: &[CosignerSignature], +) -> serde_json::Value { + let mut payload = delta_payload.clone(); + if let Some(signatures) = payload + .get_mut("signatures") + .and_then(serde_json::Value::as_array_mut) + { + signatures.retain(|entry| { + entry + .get("signer_id") + .and_then(serde_json::Value::as_str) + .is_some_and(|signer| { + approvals + .iter() + .any(|approval| approval.signer_id.eq_ignore_ascii_case(signer)) + }) + }); + } + payload +} diff --git a/crates/server/src/storage/postgres_recovery.rs b/crates/server/src/storage/postgres_recovery.rs index cd3bc9e1..974cded3 100644 --- a/crates/server/src/storage/postgres_recovery.rs +++ b/crates/server/src/storage/postgres_recovery.rs @@ -14,6 +14,7 @@ use crate::metadata::{Auth, MidenNetworkType, NetworkConfig}; use crate::schema::{account_metadata, delta_proposals, states}; use crate::storage::LeaseFence; use crate::storage::encryption::{cipher::StorageCipher, envelope::RecordAad}; +use crate::utils::normalize_commitment_hex; use super::{Pool, lease_fence_is_current, lock_account_metadata}; @@ -96,8 +97,22 @@ impl PendingOutcome { #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum ProposalChange { + Keep, Remove, - DropApprovals(DeltaStatus), + DropApprovals { + status: DeltaStatus, + delta_payload: Value, + }, +} + +impl ProposalChange { + pub(crate) fn label(&self) -> &'static str { + match self { + Self::Keep => "keep", + Self::Remove => "remove", + Self::DropApprovals { .. } => "drop_approvals", + } + } } /// A pending proposal as the scan read it, and what the restore does with it. @@ -105,7 +120,21 @@ pub(crate) enum ProposalChange { pub(crate) struct ReviewedProposal { pub commitment: String, pub status: DeltaStatus, - pub change: Option, + pub change: ProposalChange, +} + +/// The review of an account's pending proposals, or the fact that they could not be read. +/// A restore leaves unreadable proposals as they are. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ProposalReview { + Reviewed(Vec), + Unreadable, +} + +/// What a restore writes to one proposal row. +enum ProposalWrite { + Remove, + Replace { status: Value, delta_payload: Value }, } #[derive(Debug, Default, Clone, PartialEq, Eq, diesel::QueryableByName)] @@ -375,33 +404,33 @@ impl RecoveryStore { candidate: &RecoveryCandidate, state_json: &Value, auth: &Auth, - proposals: &[ReviewedProposal], + proposals: &ProposalReview, now: DateTime, lease: &Lease, ) -> Result { let account_id = &candidate.account_id; - let mut reviewed = proposals - .iter() - .map(|proposal| { - serde_json::to_value(&proposal.status) - .map(|status| (proposal.commitment.clone(), status)) - .map_err(|error| error.to_string()) - }) - .collect::, _>>()?; - reviewed.sort_by(|left, right| left.0.cmp(&right.0)); - let changes = proposals - .iter() + let reviewed = match proposals { + ProposalReview::Reviewed(reviewed) => Some(reviewed), + ProposalReview::Unreadable => None, + }; + let expected = reviewed.map(|reviewed| { + let mut expected: Vec<(String, DeltaStatus)> = reviewed + .iter() + .map(|proposal| (proposal.commitment.clone(), proposal.status.clone())) + .collect(); + expected.sort_by(|left, right| left.0.cmp(&right.0)); + expected + }); + let writes = reviewed + .into_iter() + .flatten() .filter_map(|proposal| { - let change = match proposal.change.as_ref()? { - ProposalChange::Remove => Ok(None), - ProposalChange::DropApprovals(status) => serde_json::to_value(status) - .map(Some) - .map_err(|error| error.to_string()), - }; - Some(change.map(|change| (proposal.commitment.clone(), change))) + self.proposal_write(account_id, proposal) + .map(|write| write.map(|write| (proposal.commitment.clone(), write))) + .transpose() }) .collect::, _>>()?; - let changed: Vec = changes + let changed: Vec = writes .iter() .map(|(commitment, _)| commitment.clone()) .collect(); @@ -440,7 +469,9 @@ impl RecoveryStore { if !lease_fence_is_current(conn, &fence).await? { return Ok(false); } - if pending_proposals(conn, account_id).await? != reviewed { + if let Some(expected) = &expected + && pending_proposals(conn, account_id).await? != *expected + { return Ok(false); } @@ -464,18 +495,24 @@ impl RecoveryStore { .execute(conn) .await?; - for (commitment, kept_status) in &changes { + for (commitment, write) in &writes { let proposal = delta_proposals::table .filter(delta_proposals::account_id.eq(account_id)) .filter(delta_proposals::commitment.eq(commitment)); - match kept_status { - Some(status) => { + match write { + ProposalWrite::Replace { + status, + delta_payload, + } => { diesel::update(proposal) - .set(delta_proposals::status.eq(status)) + .set(( + delta_proposals::status.eq(status), + delta_proposals::delta_payload.eq(delta_payload), + )) .execute(conn) .await? } - None => diesel::delete(proposal).execute(conn).await?, + ProposalWrite::Remove => diesel::delete(proposal).execute(conn).await?, }; } @@ -510,6 +547,41 @@ impl RecoveryStore { .await } + /// The row write for a reviewed proposal, with its payload encrypted as the storage layer + /// stores it. + fn proposal_write( + &self, + account_id: &str, + proposal: &ReviewedProposal, + ) -> Result, String> { + let (status, delta_payload) = match &proposal.change { + ProposalChange::Keep => return Ok(None), + ProposalChange::Remove => return Ok(Some(ProposalWrite::Remove)), + ProposalChange::DropApprovals { + status, + delta_payload, + } => (status, delta_payload), + }; + let commitment = normalize_commitment_hex(&proposal.commitment) + .unwrap_or_else(|_| proposal.commitment.clone()); + let delta_payload = match &self.cipher { + Some(cipher) => cipher + .encrypt( + &RecordAad::Proposal { + account_id, + commitment: &commitment, + }, + delta_payload, + ) + .map_err(|error| error.to_string())?, + None => delta_payload.clone(), + }; + Ok(Some(ProposalWrite::Replace { + status: serde_json::to_value(status).map_err(|error| error.to_string())?, + delta_payload, + })) + } + async fn write<'a, T, F>(&self, operation: &str, work: F) -> Result where T: Send + 'a, @@ -538,11 +610,12 @@ impl RecoveryStore { } } -/// The account's pending proposals and their statuses, locked for the transaction. +/// The account's pending proposals and their statuses, locked for the transaction and read +/// the way the storage layer reads them. async fn pending_proposals( conn: &mut AsyncPgConnection, account_id: &str, -) -> QueryResult> { +) -> QueryResult> { #[derive(diesel::QueryableByName)] struct PendingProposal { #[diesel(sql_type = Text)] @@ -551,7 +624,7 @@ async fn pending_proposals( status: Value, } - let mut pending: Vec<(String, Value)> = diesel::sql_query( + let mut pending: Vec<(String, DeltaStatus)> = diesel::sql_query( "SELECT commitment, status FROM delta_proposals WHERE account_id = $1 AND status_kind = 'pending' FOR UPDATE", @@ -560,7 +633,12 @@ async fn pending_proposals( .load::(conn) .await? .into_iter() - .map(|proposal| (proposal.commitment, proposal.status)) + .filter_map(|proposal| { + serde_json::from_value::(proposal.status) + .ok() + .filter(DeltaStatus::is_pending) + .map(|status| (proposal.commitment, status)) + }) .collect(); pending.sort_by(|left, right| left.0.cmp(&right.0)); Ok(pending) diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index c6172702..d3df7598 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -364,7 +364,9 @@ Start Guardian once with `GUARDIAN_STARTUP_ACCOUNT_RECOVERY=dry-run` first. A dry run performs the same checks, logs what `apply` would do with each account, and writes nothing: no account record, no report row and no progress. It takes no lease, so it runs with any number of replicas, and it -checks from the first account each time. The final log line counts the +checks from the first account each time. It names each proposal a restore +would change, and says so when `apply` would not run here (more than one +replica, or the check already completed). The final log line counts the accounts that would be restored (`restorable`), deferred and left unresolved. **Applying** @@ -403,7 +405,8 @@ A restore also reviews the account's pending proposals against the signers of the restored state. A proposal that one of those signers proposed or approved stays, keeping only their approvals. Any other pending proposal is removed. Every proposal a restore changes or removes is saved with the previous record -in `account_recovery_backups.prior_proposals`. +in `account_recovery_backups.prior_proposals`. If the account's proposals +cannot be read, the restore leaves them as they are. A paused account, or one with work in flight (a delta awaiting its on-chain outcome or a reserved execution), is deferred and checked again on a later @@ -440,9 +443,9 @@ listeners without finishing the check. **Reviewing the result** After `apply`, `account_recovery_pending` has one row for each account that -was checked and not brought up to date. `outcome` is `deferred` (a later startup checks it -again) or `unresolved` (Guardian does not hold what it needs), and `reason` -says why. When the canonical history held here does not reach back to the +was checked and not brought up to date. `outcome` is `deferred` (a later +startup checks it again) or `unresolved` (Guardian does not hold what it +needs), and `reason` says why. When the canonical history held here does not reach back to the account's creation, `history_starts_at_nonce` and `history_base_commitment` show where it begins. An unresolved account is brought up to date the next time it is registered with its current state. From 7ebf6f745b2a468a1b039866880e4433de7d1cbe Mon Sep 17 00:00:00 2001 From: Zeljko Date: Thu, 19 Nov 2026 17:39:31 +0100 Subject: [PATCH 4/8] fix(server): tighten startup recovery checks for undeployed and paused accounts - An account the configured node has no record of, with no canonical deltas held, is unchanged only when its stored state is a valid first state. Any other record is listed as unresolved. The account's network label no longer plays a part. - A restore no longer compares the stored nonce with the rebuilt one. The chain head already equals the latest canonical delta at that point. - A paused account is checked like any other and stays paused. --- crates/server/src/jobs/account_recovery.rs | 91 ++++++----------- .../jobs/account_recovery/postgres_tests.rs | 99 ++++++++++--------- .../server/src/storage/postgres_recovery.rs | 5 +- docs/PRODUCTION.md | 14 ++- 4 files changed, 95 insertions(+), 114 deletions(-) diff --git a/crates/server/src/jobs/account_recovery.rs b/crates/server/src/jobs/account_recovery.rs index d677ddc3..2e220882 100644 --- a/crates/server/src/jobs/account_recovery.rs +++ b/crates/server/src/jobs/account_recovery.rs @@ -512,7 +512,7 @@ async fn check_candidate( "the configured node has no record of the account".into(), )); } - if candidate.legacy_network_label || candidate.stored_nonce == Some(0) { + if holds_first_state(state, candidate).await? { return Ok(RecoveryOutcome::Unchanged); } } @@ -524,9 +524,15 @@ async fn check_candidate( candidate.canonical_nonce, candidate.canonical_commitment.as_deref(), ) else { - return Ok(RecoveryOutcome::Unresolved( - "no canonical history supplies a recovery target".into(), - )); + let reason = match chain { + StateVerification::Absent => { + "the configured node has no record of the account and no canonical history is held" + } + StateVerification::Match | StateVerification::Mismatch { .. } => { + "no canonical history supplies a recovery target" + } + }; + return Ok(RecoveryOutcome::Unresolved(reason.into())); }; if !matches!(&chain, StateVerification::Mismatch { on_chain } if on_chain == target) { return Ok(RecoveryOutcome::Retryable( @@ -558,47 +564,14 @@ async fn check_candidate( )); } - let stored = if candidate.stored_nonce.is_none() { - match state.storage.pull_state(&candidate.account_id).await { - Ok(stored) if stored.commitment == candidate.stored_commitment => Some(stored), - Ok(_) => { - return Ok(RecoveryOutcome::Retryable( - "stored commitment changed during recovery".into(), - )); - } - Err(reason) if is_record_corruption(&reason) => { - return Ok(RecoveryOutcome::Unresolved(reason)); - } - Err(reason) => return Err(reason), - } - } else { - None - }; - let stored_nonce = candidate - .stored_nonce - .and_then(|nonce| u64::try_from(nonce).ok()); let network = state.network_client.clone(); let account_id = candidate.account_id.clone(); let target = target.to_owned(); let proof = reconstructor() - .run_background(move || { - prove_recovery( - network.as_ref(), - &account_id, - &target, - deltas, - stored_nonce, - stored, - ) - }) + .run_background(move || prove_recovery(network.as_ref(), &account_id, &target, deltas)) .await; let (applied, auth) = match proof { - Ok(Some(proof)) => proof, - Ok(None) => { - return Ok(RecoveryOutcome::Unresolved( - "the stored state is newer than the verified history".into(), - )); - } + Ok(proof) => proof, Err(ReconstructError::Operation(reason)) => return Ok(RecoveryOutcome::Unresolved(reason)), Err(ReconstructError::Task(reason)) => { return Err(format!("state reconstruction failed: {reason}")); @@ -645,31 +618,31 @@ async fn check_candidate( }))) } +/// Whether the stored state is a first state, which stands without a record on the network. +async fn holds_first_state( + state: &AppState, + candidate: &RecoveryCandidate, +) -> Result { + if candidate.stored_nonce.is_some_and(|nonce| nonce != 0) { + return Ok(false); + } + match state.storage.pull_state(&candidate.account_id).await { + Ok(stored) => Ok(state + .network_client + .validate_first_state(&stored.state_json) + .is_ok()), + Err(reason) if is_record_corruption(&reason) => Ok(false), + Err(reason) => Err(reason), + } +} + fn prove_recovery( network: &dyn NetworkClient, account_id: &str, target: &str, deltas: Vec, - stored_nonce: Option, - stored: Option, -) -> Result, String> { +) -> Result<(AppliedState, Auth), String> { let applied = reconstruct(network, account_id, target, deltas)?; - let nonce = match stored { - Some(stored) => { - let head = network.get_state_head(account_id, &stored.state_json)?; - if head.commitment != stored.commitment { - return Err("stored state does not match its recorded commitment".into()); - } - head.nonce - } - None => stored_nonce, - } - .ok_or("stored state has no verifiable nonce")?; - let recovered_nonce = applied.nonce.ok_or("recovered state has no nonce")?; - if nonce > recovered_nonce { - return Ok(None); - } - let binding = network .account_auth_binding(&applied.state_json)? .ok_or("missing account auth binding")?; @@ -678,7 +651,7 @@ fn prove_recovery( .guardian_scheme .ok_or("missing on-chain Guardian signature scheme")?; let auth = recovered_auth(scheme, &applied)?; - Ok(Some((applied, auth))) + Ok((applied, auth)) } fn reconstruct( diff --git a/crates/server/src/jobs/account_recovery/postgres_tests.rs b/crates/server/src/jobs/account_recovery/postgres_tests.rs index 09ea9c95..c9255d30 100644 --- a/crates/server/src/jobs/account_recovery/postgres_tests.rs +++ b/crates/server/src/jobs/account_recovery/postgres_tests.rs @@ -121,15 +121,11 @@ async fn rejects_incomplete_history_and_commitment_mismatches() { reconstruct(&network, &account_id, target, deltas.clone()).unwrap(), expected ); - assert!( - prove_recovery(&network, &account_id, target, deltas.clone(), Some(1), None) - .unwrap() - .is_some() - ); - assert!( - prove_recovery(&network, &account_id, target, deltas.clone(), Some(3), None) + assert_eq!( + prove_recovery(&network, &account_id, target, deltas.clone()) .unwrap() - .is_none() + .0, + expected ); assert!(reconstruct(&network, &account_id, target, vec![deltas[0].clone()]).is_err()); deltas[1].new_commitment = Some("wrong commitment".into()); @@ -181,10 +177,7 @@ async fn rebuilds_the_signer_set_under_each_scheme() { &account.id().to_hex(), &rebuilt.commitment, deltas, - Some(1), - None, ) - .unwrap() .unwrap(); assert_eq!(auth, expected); } @@ -435,7 +428,6 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e let candidate = candidates.pop().unwrap(); assert_eq!(candidate.canonical_nonce, Some(2)); assert_eq!(candidate.stored_nonce, None); - assert!(candidate.legacy_network_label); let mut absent = state.clone(); absent.network_client = Arc::new( @@ -453,35 +445,49 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e .unwrap() .pop() .unwrap(); - assert!(!undeployed.legacy_network_label); - undeployed.stored_nonce = Some(0); undeployed.canonical_nonce = None; undeployed.canonical_commitment = None; - assert!(matches!( - check_candidate(&absent, &undeployed).await.unwrap(), - RecoveryOutcome::Unchanged - )); - let mut labelled_without_history = store - .candidates(MidenNetworkType::Testnet, "", 10) - .await - .unwrap() - .pop() - .unwrap(); - assert!(labelled_without_history.legacy_network_label); - labelled_without_history.stored_nonce = Some(5); - labelled_without_history.canonical_nonce = None; - labelled_without_history.canonical_commitment = None; - let mut still_absent = state.clone(); - still_absent.network_client = Arc::new( - crate::testing::mocks::MockNetworkClient::new() - .with_verify_commitment(Ok(StateVerification::Absent)), - ); - assert!(matches!( - check_candidate(&still_absent, &labelled_without_history) + let absent_with = |first_state: Result<(), String>| { + let mut absent = state.clone(); + absent.network_client = Arc::new( + crate::testing::mocks::MockNetworkClient::new() + .with_verify_commitment(Ok(StateVerification::Absent)) + .with_validate_first_state(first_state), + ); + absent + }; + for stored_nonce in [Some(0), None] { + undeployed.stored_nonce = stored_nonce; + assert!(matches!( + check_candidate(&absent_with(Ok(())), &undeployed) + .await + .unwrap(), + RecoveryOutcome::Unchanged + )); + assert!(matches!( + check_candidate(&absent_with(Err("nonce is 5, not 0".into())), &undeployed) + .await + .unwrap(), + RecoveryOutcome::Unresolved(reason) if reason.contains("no record of the account") + )); + } + for network_type in [MidenNetworkType::Local, MidenNetworkType::Testnet] { + let mut past_first_state = store + .candidates(network_type, "", 10) .await - .unwrap(), - RecoveryOutcome::Unchanged - )); + .unwrap() + .pop() + .unwrap(); + past_first_state.stored_nonce = Some(5); + past_first_state.canonical_nonce = None; + past_first_state.canonical_commitment = None; + assert!(matches!( + check_candidate(&absent_with(Ok(())), &past_first_state) + .await + .unwrap(), + RecoveryOutcome::Unresolved(reason) if reason.contains("no record of the account") + )); + } { let mut conn = metadata.pool_handle().get().await.unwrap(); @@ -524,7 +530,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e ahead.stored_nonce = Some(9); assert!(matches!( check_candidate(&state, &ahead).await.unwrap(), - RecoveryOutcome::Unresolved(reason) if reason.contains("newer than the verified history") + RecoveryOutcome::Verified(_) )); let pending = DeltaObject { nonce: 3, @@ -828,6 +834,10 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e .await .unwrap(); store.leader.release(reviewing).await.unwrap(); + metadata + .set_pause(account_id, state.clock.now(), "operator review") + .await + .unwrap(); run_once(&state, &store, MidenNetworkType::Testnet) .await @@ -931,14 +941,11 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e .commitment, target ); + let restored_metadata = metadata.get_for_read(account_id).await.unwrap().unwrap(); + assert_eq!(restored_metadata.auth, expected_auth); assert_eq!( - metadata - .get_for_read(account_id) - .await - .unwrap() - .unwrap() - .auth, - expected_auth + restored_metadata.paused_reason.as_deref(), + Some("operator review") ); #[derive(diesel::QueryableByName)] diff --git a/crates/server/src/storage/postgres_recovery.rs b/crates/server/src/storage/postgres_recovery.rs index 974cded3..394fcb8b 100644 --- a/crates/server/src/storage/postgres_recovery.rs +++ b/crates/server/src/storage/postgres_recovery.rs @@ -20,7 +20,6 @@ use super::{Pool, lease_fence_is_current, lock_account_metadata}; const UNSETTLED_ACCOUNT_WORK: &str = " CASE - WHEN m.paused_at IS NOT NULL THEN 'the account is paused' WHEN m.has_pending_candidate OR EXISTS ( SELECT 1 FROM deltas pending WHERE pending.account_id = st.account_id AND pending.status_kind IN ('candidate', 'retained') @@ -72,8 +71,6 @@ pub(crate) struct RecoveryCandidate { pub history_starts_at_nonce: Option, #[diesel(sql_type = Nullable)] pub history_base_commitment: Option, - #[diesel(sql_type = Bool)] - pub legacy_network_label: bool, #[diesel(sql_type = Timestamptz)] state_updated_at: DateTime, #[diesel(sql_type = BigInt)] @@ -288,7 +285,7 @@ impl RecoveryStore { "SELECT st.account_id, st.nonce AS stored_nonce, st.commitment AS stored_commitment, d.nonce AS canonical_nonce, ({UNSETTLED_ACCOUNT_WORK}) AS unsettled_work, d.new_commitment AS canonical_commitment, st.updated_at AS state_updated_at, - m.auth_version, m.network_config <> $2 AS legacy_network_label, + m.auth_version, first_held.nonce AS history_starts_at_nonce, first_held.prev_commitment AS history_base_commitment {CANDIDATE_SOURCE} diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index d3df7598..2c5b0d9e 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -392,6 +392,9 @@ record differs from its latest canonical delta, Guardian asks the configured Miden node for the account's current commitment. - If the stored record already matches the network, nothing changes. +- If the configured node has no record of the account and Guardian holds no + canonical deltas for it, the stored record stands only when it is the + account's first state. Any other record is listed as unresolved. - If Guardian holds the account's canonical deltas from creation (up to 128), it rebuilds the account from them. When the result matches the network and is bound to this server's ACK key, Guardian stores it together with its @@ -408,9 +411,9 @@ Every proposal a restore changes or removes is saved with the previous record in `account_recovery_backups.prior_proposals`. If the account's proposals cannot be read, the restore leaves them as they are. -A paused account, or one with work in flight (a delta awaiting its on-chain -outcome or a reserved execution), is deferred and checked again on a later -startup. +An account with work in flight (a delta awaiting its on-chain outcome or a +reserved execution) is deferred and checked again on a later startup. A paused +account is checked like any other, and stays paused. **How long it takes** @@ -447,8 +450,9 @@ was checked and not brought up to date. `outcome` is `deferred` (a later startup checks it again) or `unresolved` (Guardian does not hold what it needs), and `reason` says why. When the canonical history held here does not reach back to the account's creation, `history_starts_at_nonce` and `history_base_commitment` -show where it begins. An unresolved account is brought up to date the next -time it is registered with its current state. +show where it begins. An unresolved account the network has a +record of is brought up to date the next time it is registered with its +current state. A row is removed once its account is brought up to date or found unchanged. A row whose `stored_commitment` no longer matches the account's stored state is From 4724046abcd6e37a86bc5a5ad2dadc14fde59e5a Mon Sep 17 00:00:00 2001 From: Zeljko Date: Thu, 19 Nov 2026 17:52:53 +0100 Subject: [PATCH 5/8] test(server): cover a restore over a higher stored nonce and a kept pause - The restore test now runs over a stored nonce above the rebuilt one and checks the nonce it ends at, and checks that the pause time is kept. - Drop a repeated check of an account the node has no record of. - Docs: such an account is deferred while it has work in flight. --- .../jobs/account_recovery/postgres_tests.rs | 50 +++++++++---------- docs/PRODUCTION.md | 3 +- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/crates/server/src/jobs/account_recovery/postgres_tests.rs b/crates/server/src/jobs/account_recovery/postgres_tests.rs index c9255d30..4c36e5af 100644 --- a/crates/server/src/jobs/account_recovery/postgres_tests.rs +++ b/crates/server/src/jobs/account_recovery/postgres_tests.rs @@ -471,23 +471,13 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e RecoveryOutcome::Unresolved(reason) if reason.contains("no record of the account") )); } - for network_type in [MidenNetworkType::Local, MidenNetworkType::Testnet] { - let mut past_first_state = store - .candidates(network_type, "", 10) + undeployed.stored_nonce = Some(5); + assert!(matches!( + check_candidate(&absent_with(Ok(())), &undeployed) .await - .unwrap() - .pop() - .unwrap(); - past_first_state.stored_nonce = Some(5); - past_first_state.canonical_nonce = None; - past_first_state.canonical_commitment = None; - assert!(matches!( - check_candidate(&absent_with(Ok(())), &past_first_state) - .await - .unwrap(), - RecoveryOutcome::Unresolved(reason) if reason.contains("no record of the account") - )); - } + .unwrap(), + RecoveryOutcome::Unresolved(reason) if reason.contains("no record of the account") + )); { let mut conn = metadata.pool_handle().get().await.unwrap(); @@ -521,17 +511,6 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e RecoveryOutcome::Retryable(reason) if reason.contains("current chain head") )); chain.set_on_chain_commitment(account_id, target); - let mut ahead = store - .candidates(MidenNetworkType::Testnet, "", 10) - .await - .unwrap() - .pop() - .unwrap(); - ahead.stored_nonce = Some(9); - assert!(matches!( - check_candidate(&state, &ahead).await.unwrap(), - RecoveryOutcome::Verified(_) - )); let pending = DeltaObject { nonce: 3, prev_commitment: target.into(), @@ -838,6 +817,18 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e .set_pause(account_id, state.clock.now(), "operator review") .await .unwrap(); + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("UPDATE states SET nonce = 9 WHERE account_id = $1") + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + } + assert_eq!( + storage.pull_state_head(account_id).await.unwrap().nonce, + Some(9) + ); run_once(&state, &store, MidenNetworkType::Testnet) .await @@ -943,6 +934,11 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e ); let restored_metadata = metadata.get_for_read(account_id).await.unwrap().unwrap(); assert_eq!(restored_metadata.auth, expected_auth); + assert!(restored_metadata.paused_at.is_some()); + assert_eq!( + storage.pull_state_head(account_id).await.unwrap().nonce, + Some(2) + ); assert_eq!( restored_metadata.paused_reason.as_deref(), Some("operator review") diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index 2c5b0d9e..1782a0e1 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -394,7 +394,8 @@ Miden node for the account's current commitment. - If the stored record already matches the network, nothing changes. - If the configured node has no record of the account and Guardian holds no canonical deltas for it, the stored record stands only when it is the - account's first state. Any other record is listed as unresolved. + account's first state. Any other record is listed as unresolved, or + deferred while the account has work in flight. - If Guardian holds the account's canonical deltas from creation (up to 128), it rebuilds the account from them. When the result matches the network and is bound to this server's ACK key, Guardian stores it together with its From cf6b153a786bca37c9c7dc55d4367845b0479aba Mon Sep 17 00:00:00 2001 From: Zeljko Date: Thu, 19 Nov 2026 18:37:34 +0100 Subject: [PATCH 6/8] fix(server): startup account recovery review follow-ups - Rebuild an account from its canonical deltas a page at a time, with no cap on their number and a one minute limit per account. - Defer an account only for work in flight that starts from its latest canonical commitment, or for a delta that ends at the network's state. - List an account as unresolved when the network's commitment is not in the canonical history held here, and defer it when the node reports an earlier state. - A scan pass can span several startups: saved progress keeps the pass's counts, and a pass that deferred an account is followed by a new one. - A restore removes the account's pending record in its own transaction, reviews proposals before its last network check, keeps a proposal whose transaction summary cannot be read, and ignores proposal rows with an unreadable status when it compares them. - Docs: removing a stored state by its listed commitment, and the ECS deploy steps for the check. --- .../up.sql | 4 +- crates/server/src/jobs/account_recovery.rs | 221 +++++++---- .../jobs/account_recovery/postgres_tests.rs | 367 ++++++++++++++++-- .../src/jobs/account_recovery/proposals.rs | 16 +- crates/server/src/schema.rs | 2 + .../server/src/storage/postgres_recovery.rs | 128 +++++- docs/PRODUCTION.md | 61 ++- infra/README.md | 2 +- infra/terraform.tfvars.example | 2 +- infra/variables.tf | 4 +- 10 files changed, 661 insertions(+), 146 deletions(-) diff --git a/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql index 51c4f0ec..e52834fa 100644 --- a/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql +++ b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql @@ -2,5 +2,7 @@ CREATE TABLE account_recovery_startup ( network_config JSONB PRIMARY KEY, resume_after VARCHAR(128) NOT NULL DEFAULT '', completed_at TIMESTAMPTZ, - unresolved_accounts BIGINT + unresolved_accounts BIGINT, + pass_unresolved BIGINT NOT NULL DEFAULT 0, + pass_deferred BOOLEAN NOT NULL DEFAULT FALSE ); diff --git a/crates/server/src/jobs/account_recovery.rs b/crates/server/src/jobs/account_recovery.rs index 2e220882..bf7be759 100644 --- a/crates/server/src/jobs/account_recovery.rs +++ b/crates/server/src/jobs/account_recovery.rs @@ -13,14 +13,15 @@ use crate::network::{ AppliedState, NetworkClient, ReconstructError, RpcReadMode, StateVerification, reconstructor, }; use crate::state::AppState; -use crate::storage::is_record_corruption; use crate::storage::postgres::recovery::{ PendingOutcome, ProposalChange, ProposalReview, RecoveryCandidate, RecoveryStore, ReviewedProposal, ScanProgress, }; +use crate::storage::{AccountDeltaCursor, is_record_corruption}; const PAGE_SIZE: i64 = 100; -const MAX_DELTAS: u32 = 128; +const HISTORY_PAGE: u32 = 128; +const REBUILD_BUDGET: Duration = Duration::from_secs(60); const LEASE_TTL: Duration = Duration::from_secs(30); const SCAN_BUDGET: Duration = Duration::from_secs(300); const REMAINING_COUNT_BUDGET: Duration = Duration::from_secs(1); @@ -29,6 +30,14 @@ const STALE_LEASE_WAIT: Duration = Duration::from_secs(35); const LEASE_RETRY_INTERVAL: Duration = Duration::from_secs(2); const MAX_CONSECUTIVE_FAILED_CHECKS: usize = 3; +/// An account rebuilt from its canonical history, or the reason that history does not +/// rebuild it. +#[derive(Debug)] +enum Rebuild { + Rebuilt(AppliedState), + Unresolved(String), +} + #[derive(Debug)] enum RecoveryOutcome { Unchanged, @@ -79,6 +88,7 @@ impl Restoration { enum PendingRecord { Clear, + ClearedByRestore, Write(PendingOutcome, String), } @@ -109,23 +119,24 @@ enum ScanStep { } impl ScanSummary { - /// What a scan that started at `started` leaves for the next startup: its position when - /// it stopped early, and otherwise a pass from the first account. Only a pass that - /// started at the first account, reached the last one within one startup and deferred - /// none completes the scan. + /// What a scan that started at `started` leaves for the next startup. A scan that stops + /// early keeps its position and the counts of its pass. A pass that reaches the last + /// account completes the scan when it deferred no account, and is followed by a new pass + /// from the first account otherwise. fn next_step(&self, started: ScanProgress) -> ScanStep { + let unresolved = started.unresolved + self.unresolved; + let deferred = started.deferred || self.retryable > 0 || self.failed > 0; if self.stopped_early { return ScanStep::Resume(ScanProgress { resume_after: self.last_checked.clone().unwrap_or(started.resume_after), + unresolved, + deferred, }); } - let covered_every_account = started.resume_after.is_empty(); - if covered_every_account && self.retryable == 0 && self.failed == 0 { - ScanStep::Completed { - unresolved: self.unresolved, - } - } else { + if deferred { ScanStep::Resume(ScanProgress::default()) + } else { + ScanStep::Completed { unresolved } } } } @@ -330,7 +341,8 @@ async fn run( if cancel.is_cancelled() { return Err("account recovery lease lost".into()); } - let result = match timeout_at(deadline, check_candidate(state, &candidate)).await { + let result = match timeout_at(deadline, check_candidate(state, store, &candidate)).await + { Ok(result) => result, Err(_) => { summary.stopped_early = true; @@ -381,7 +393,7 @@ async fn run( "Restored chain-verified account state and signers" ); restoration.log_proposal_changes(&candidate.account_id, false); - PendingRecord::Clear + PendingRecord::ClearedByRestore } Ok(false) => { summary.retryable += 1; @@ -424,6 +436,7 @@ async fn run( }; if let ScanWrites::Apply(lease) = writes { let recorded = match record { + PendingRecord::ClearedByRestore => Ok(()), PendingRecord::Clear => { bounded_db_call( "clear a pending recovery record", @@ -493,6 +506,7 @@ async fn run( async fn check_candidate( state: &AppState, + store: &RecoveryStore, candidate: &RecoveryCandidate, ) -> Result { let chain = state @@ -519,6 +533,16 @@ async fn check_candidate( if let Some(unsettled_work) = &candidate.unsettled_work { return Ok(RecoveryOutcome::Retryable(unsettled_work.clone())); } + if let StateVerification::Mismatch { on_chain } = &chain + && candidate.canonical_commitment.as_deref() != Some(on_chain.as_str()) + && store + .holds_landed_delta(&candidate.account_id, on_chain) + .await? + { + return Ok(RecoveryOutcome::Retryable( + "a delta that landed awaits its promotion".into(), + )); + } let (Some(canonical_nonce), Some(target)) = ( candidate.canonical_nonce, @@ -534,41 +558,46 @@ async fn check_candidate( }; return Ok(RecoveryOutcome::Unresolved(reason.into())); }; - if !matches!(&chain, StateVerification::Mismatch { on_chain } if on_chain == target) { - return Ok(RecoveryOutcome::Retryable( - "latest canonical delta does not match the current chain head".into(), - )); + match &chain { + StateVerification::Mismatch { on_chain } if on_chain == target => {} + StateVerification::Mismatch { on_chain } => { + let earlier = store + .holds_canonical_commitment(&candidate.account_id, on_chain) + .await?; + return Ok(if earlier { + RecoveryOutcome::Retryable( + "the configured node reports an earlier state of the account".into(), + ) + } else { + RecoveryOutcome::Unresolved( + "the configured node's commitment is missing from the canonical history held here".into(), + ) + }); + } + StateVerification::Match | StateVerification::Absent => { + return Ok(RecoveryOutcome::Retryable( + "latest canonical delta does not match the current chain head".into(), + )); + } } - let Some(count) = u32::try_from(canonical_nonce) - .ok() - .filter(|count| (1..=MAX_DELTAS).contains(count)) - else { - return Ok(RecoveryOutcome::Unresolved( - "canonical history exceeds the recovery limit".into(), - )); - }; - let history = state - .storage - .list_canonical_deltas_paged(&candidate.account_id, count, None) - .await; - let deltas = match history { - Ok(deltas) => deltas, - Err(reason) if is_record_corruption(&reason) => { - return Ok(RecoveryOutcome::Unresolved(reason)); + let rebuild = rebuild(state, &candidate.account_id, canonical_nonce, HISTORY_PAGE); + let rebuilt = match timeout(REBUILD_BUDGET, rebuild).await { + Ok(rebuilt) => match rebuilt? { + Rebuild::Rebuilt(rebuilt) => rebuilt, + Rebuild::Unresolved(reason) => return Ok(RecoveryOutcome::Unresolved(reason)), + }, + Err(_) => { + return Ok(RecoveryOutcome::Unresolved( + "the canonical history takes too long to rebuild".into(), + )); } - Err(reason) => return Err(reason), }; - if deltas.len() != count as usize { - return Ok(RecoveryOutcome::Unresolved( - "missing canonical deltas from account creation".into(), - )); - } let network = state.network_client.clone(); let account_id = candidate.account_id.clone(); let target = target.to_owned(); let proof = reconstructor() - .run_background(move || prove_recovery(network.as_ref(), &account_id, &target, deltas)) + .run_background(move || prove_recovery(network.as_ref(), &account_id, &target, rebuilt)) .await; let (applied, auth) = match proof { Ok(proof) => proof, @@ -585,6 +614,16 @@ async fn check_candidate( )); } + let proposals = match state + .storage + .pull_pending_proposals(&candidate.account_id) + .await + { + Ok(pending) => ProposalReview::Reviewed(proposals::review(&applied, &pending)), + Err(reason) if is_record_corruption(&reason) => ProposalReview::Unreadable, + Err(reason) => return Err(reason), + }; + match state .network_client .verify_commitment( @@ -602,15 +641,6 @@ async fn check_candidate( } } - let proposals = match state - .storage - .pull_pending_proposals(&candidate.account_id) - .await - { - Ok(pending) => ProposalReview::Reviewed(proposals::review(&applied, &pending)), - Err(reason) if is_record_corruption(&reason) => ProposalReview::Unreadable, - Err(reason) => return Err(reason), - }; Ok(RecoveryOutcome::Verified(Box::new(Restoration { state: applied, auth, @@ -636,13 +666,74 @@ async fn holds_first_state( } } +/// Rebuilds the account from its canonical deltas, read `page` at a time from the first one. +async fn rebuild( + state: &AppState, + account_id: &str, + canonical_nonce: i64, + page: u32, +) -> Result { + let mut rebuilt: Option = None; + let mut first = 1_i64; + let Some(end) = canonical_nonce.checked_add(1) else { + return Ok(Rebuild::Unresolved( + "canonical nonce exceeds the storage range".into(), + )); + }; + while first < end { + let after = first.saturating_add(i64::from(page)).min(end); + let expected = u32::try_from(after - first).map_err(|error| error.to_string())?; + let read = state + .storage + .list_canonical_deltas_paged( + account_id, + expected, + Some(AccountDeltaCursor { last_nonce: after }), + ) + .await; + let deltas = match read { + Ok(deltas) => deltas, + Err(reason) if is_record_corruption(&reason) => { + return Ok(Rebuild::Unresolved(reason)); + } + Err(reason) => return Err(reason), + }; + if u32::try_from(deltas.len()) != Ok(expected) { + return Ok(Rebuild::Unresolved( + "missing canonical deltas from account creation".into(), + )); + } + let network = state.network_client.clone(); + let account = account_id.to_owned(); + let replayed = reconstructor() + .run_background(move || replay(network.as_ref(), &account, rebuilt, deltas)) + .await; + rebuilt = match replayed { + Ok(applied) => Some(applied), + Err(ReconstructError::Operation(reason)) => return Ok(Rebuild::Unresolved(reason)), + Err(ReconstructError::Task(reason)) => { + return Err(format!("state reconstruction failed: {reason}")); + } + }; + first = after; + } + Ok(match rebuilt { + Some(rebuilt) => Rebuild::Rebuilt(rebuilt), + None => Rebuild::Unresolved("canonical history is empty".into()), + }) +} + +/// The rebuilt state with the signer set it carries, once it is the expected account head. fn prove_recovery( network: &dyn NetworkClient, account_id: &str, target: &str, - deltas: Vec, + applied: AppliedState, ) -> Result<(AppliedState, Auth), String> { - let applied = reconstruct(network, account_id, target, deltas)?; + let head = network.get_state_head(account_id, &applied.state_json)?; + if head.commitment != target || head.nonce != applied.nonce { + return Err("reconstructed state does not match the expected account head".into()); + } let binding = network .account_auth_binding(&applied.state_json)? .ok_or("missing account auth binding")?; @@ -654,18 +745,21 @@ fn prove_recovery( Ok((applied, auth)) } -fn reconstruct( +/// Applies canonical deltas, newest first as storage returns them, on top of the state rebuilt +/// so far. Without one, the first delta creates the account. +fn replay( network: &dyn NetworkClient, account_id: &str, - target: &str, + rebuilt: Option, deltas: Vec, ) -> Result { - let mut reconstructed: Option = None; - for (index, delta) in deltas.into_iter().rev().enumerate() { - if delta.account_id != account_id - || delta.nonce != index as u64 + 1 - || !delta.status.is_canonical() - { + let first_nonce = match &rebuilt { + Some(rebuilt) => rebuilt.nonce.ok_or("rebuilt state has no nonce")? + 1, + None => 1, + }; + let mut reconstructed = rebuilt; + for (nonce, delta) in (first_nonce..).zip(deltas.into_iter().rev()) { + if delta.account_id != account_id || delta.nonce != nonce || !delta.status.is_canonical() { return Err("canonical history is not contiguous from account creation".into()); } let applied = match reconstructed { @@ -683,12 +777,7 @@ fn reconstruct( } reconstructed = Some(applied); } - let applied = reconstructed.ok_or("canonical history is empty")?; - let head = network.get_state_head(account_id, &applied.state_json)?; - if head.commitment != target || head.nonce != applied.nonce { - return Err("reconstructed state does not match the expected account head".into()); - } - Ok(applied) + reconstructed.ok_or_else(|| "canonical history is empty".into()) } fn recovered_auth(scheme: SignatureScheme, state: &AppliedState) -> Result { diff --git a/crates/server/src/jobs/account_recovery/postgres_tests.rs b/crates/server/src/jobs/account_recovery/postgres_tests.rs index 4c36e5af..f09a8b55 100644 --- a/crates/server/src/jobs/account_recovery/postgres_tests.rs +++ b/crates/server/src/jobs/account_recovery/postgres_tests.rs @@ -104,6 +104,16 @@ fn canonical_history( (vec![second, first], second_state) } +fn prove( + network: &dyn NetworkClient, + account_id: &str, + target: &str, + deltas: Vec, +) -> Result<(AppliedState, Auth), String> { + replay(network, account_id, None, deltas) + .and_then(|rebuilt| prove_recovery(network, account_id, target, rebuilt)) +} + #[tokio::test] async fn rejects_incomplete_history_and_commitment_mismatches() { let network = MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal); @@ -118,18 +128,36 @@ async fn rejects_incomplete_history_and_commitment_mismatches() { let account_id = account.id().to_hex(); let target = &expected.commitment; assert_eq!( - reconstruct(&network, &account_id, target, deltas.clone()).unwrap(), + prove(&network, &account_id, target, deltas.clone()) + .unwrap() + .0, expected ); + let first_page = replay(&network, &account_id, None, vec![deltas[1].clone()]).unwrap(); + assert_eq!(first_page.nonce, Some(1)); + assert!(prove_recovery(&network, &account_id, target, first_page.clone()).is_err()); + assert!( + replay( + &network, + &account_id, + Some(first_page.clone()), + vec![deltas[1].clone()] + ) + .is_err() + ); assert_eq!( - prove_recovery(&network, &account_id, target, deltas.clone()) - .unwrap() - .0, + replay( + &network, + &account_id, + Some(first_page), + vec![deltas[0].clone()] + ) + .unwrap(), expected ); - assert!(reconstruct(&network, &account_id, target, vec![deltas[0].clone()]).is_err()); + assert!(prove(&network, &account_id, target, vec![deltas[0].clone()]).is_err()); deltas[1].new_commitment = Some("wrong commitment".into()); - assert!(reconstruct(&network, &account_id, target, deltas).is_err()); + assert!(prove(&network, &account_id, target, deltas).is_err()); } #[tokio::test] @@ -172,7 +200,7 @@ async fn rebuilds_the_signer_set_under_each_scheme() { (Word::from([5, 6, 7, 8_u32]), guardian_scheme), ); let (deltas, rebuilt) = canonical_history(&account, &network); - let (_, auth) = prove_recovery( + let (_, auth) = prove( &network, &account.id().to_hex(), &rebuilt.commitment, @@ -244,12 +272,15 @@ fn a_proposal_review_keeps_what_the_restored_signers_proposed_or_approved() { }; let own = approval(&owner); let other = approval(&other_key); + let mut unreadable = record("unreadable", &other_key, std::slice::from_ref(&other)); + unreadable.proposal.delta_payload = serde_json::json!({ "tx_summary": "not a summary" }); let records = [ record("signed", &owner, std::slice::from_ref(&own)), record("mixed", &owner, &[own.clone(), other.clone()]), record("approved", &other_key, std::slice::from_ref(&own)), record("unrelated", &other_key, std::slice::from_ref(&other)), record("unsigned", &owner, &[]), + unreadable, ]; let changes: Vec = @@ -268,6 +299,7 @@ fn a_proposal_review_keeps_what_the_restored_signers_proposed_or_approved() { ProposalChange::Keep, ProposalChange::Remove, ProposalChange::Keep, + ProposalChange::Keep, ] ); @@ -436,7 +468,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e .with_verify_commitment(Ok(StateVerification::Absent)), ); assert!(matches!( - check_candidate(&absent, &candidate).await.unwrap(), + check_candidate(&absent, &store, &candidate).await.unwrap(), RecoveryOutcome::Retryable(reason) if reason.contains("no record of the account") )); let mut undeployed = store @@ -459,13 +491,13 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e for stored_nonce in [Some(0), None] { undeployed.stored_nonce = stored_nonce; assert!(matches!( - check_candidate(&absent_with(Ok(())), &undeployed) + check_candidate(&absent_with(Ok(())), &store, &undeployed) .await .unwrap(), RecoveryOutcome::Unchanged )); assert!(matches!( - check_candidate(&absent_with(Err("nonce is 5, not 0".into())), &undeployed) + check_candidate(&absent_with(Err("nonce is 5, not 0".into())), &store, &undeployed) .await .unwrap(), RecoveryOutcome::Unresolved(reason) if reason.contains("no record of the account") @@ -473,7 +505,7 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e } undeployed.stored_nonce = Some(5); assert!(matches!( - check_candidate(&absent_with(Ok(())), &undeployed) + check_candidate(&absent_with(Ok(())), &store, &undeployed) .await .unwrap(), RecoveryOutcome::Unresolved(reason) if reason.contains("no record of the account") @@ -505,11 +537,33 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e assert_eq!(timed_out.checked, 0); } + for page in [1, 2, HISTORY_PAGE] { + assert!(matches!( + rebuild(&state, account_id, 2, page).await.unwrap(), + Rebuild::Rebuilt(rebuilt) if rebuilt.commitment == target && rebuilt.nonce == Some(2) + )); + } + assert!(matches!( + rebuild(&state, account_id, 3, 1).await.unwrap(), + Rebuild::Unresolved(reason) if reason.contains("not contiguous") + )); + chain.set_on_chain_commitment(account_id, "different head"); assert!(matches!( - check_candidate(&state, &candidate).await.unwrap(), - RecoveryOutcome::Retryable(reason) if reason.contains("current chain head") + check_candidate(&state, &store, &candidate).await.unwrap(), + RecoveryOutcome::Unresolved(reason) if reason.contains("missing from the canonical history") )); + assert_ne!(deltas[1].prev_commitment, prior.commitment); + for earlier in [ + deltas[1].new_commitment.as_deref().unwrap(), + deltas[1].prev_commitment.as_str(), + ] { + chain.set_on_chain_commitment(account_id, earlier); + assert!(matches!( + check_candidate(&state, &store, &candidate).await.unwrap(), + RecoveryOutcome::Retryable(reason) if reason.contains("earlier state") + )); + } chain.set_on_chain_commitment(account_id, target); let pending = DeltaObject { nonce: 3, @@ -519,8 +573,20 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e ..deltas[0].clone() }; storage.submit_delta(&pending).await.unwrap(); + let current_candidate = || async { + store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap() + .pop() + .unwrap() + }; + assert_eq!( + current_candidate().await.unsettled_work.as_deref(), + Some("a delta awaits its on-chain outcome") + ); assert!(matches!( - check_candidate(&state, &candidate).await.unwrap(), + check_candidate(&state, &store, &candidate).await.unwrap(), RecoveryOutcome::Verified(_) )); assert!( @@ -571,6 +637,70 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e ) .await .unwrap(); + storage + .submit_delta(&DeltaObject { + nonce: 4, + prev_commitment: prior.commitment.clone(), + new_commitment: None, + status: DeltaStatus::candidate("2026-11-14T12:41:00Z".into()), + ..deltas[0].clone() + }) + .await + .unwrap(); + assert_eq!(current_candidate().await.unsettled_work, None); + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "INSERT INTO execution_reservations + (account_id, proposal_id, attempt, holder_id, lease_name, fence_token, + lease_expires_at, phase) + VALUES ($1, '0xreserved', 1, 'holder', 'execution', 1, now(), 'accepted')", + ) + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + diesel::sql_query( + "INSERT INTO execution_submissions + (account_id, proposal_id, attempt, candidate_nonce, transaction_id, + expected_commitment, reference_block, expiration_block, base_commitment) + VALUES ($1, '0xreserved', 1, 3, '0xtx', '0xexpected', 1, 2, $2)", + ) + .bind::(account_id) + .bind::(target) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!( + current_candidate().await.unsettled_work.as_deref(), + Some("an execution is reserved") + ); + diesel::sql_query( + "UPDATE execution_submissions SET base_commitment = $2 WHERE account_id = $1", + ) + .bind::(account_id) + .bind::(&prior.commitment) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(current_candidate().await.unsettled_work, None); + + diesel::sql_query( + "UPDATE deltas SET new_commitment = '0xlanded' WHERE account_id = $1 AND nonce = 4", + ) + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + chain.set_on_chain_commitment(account_id, "0xlanded"); + assert!(matches!( + check_candidate(&state, &store, ¤t_candidate().await) + .await + .unwrap(), + RecoveryOutcome::Retryable(reason) if reason.contains("landed") + )); + chain.set_on_chain_commitment(account_id, target); + } assert!( store @@ -778,8 +908,9 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e .unwrap() .pop() .unwrap(); - let RecoveryOutcome::Verified(reviewed) = - check_candidate(&state, &reviewed_candidate).await.unwrap() + let RecoveryOutcome::Verified(reviewed) = check_candidate(&state, &store, &reviewed_candidate) + .await + .unwrap() else { panic!("the account is restorable"); }; @@ -829,6 +960,128 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e storage.pull_state_head(account_id).await.unwrap().nonce, Some(9) ); + let eligible = store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap() + .pop() + .unwrap(); + assert!( + !store + .apply( + &eligible, + &reviewed.state.state_json, + &reviewed.auth, + &reviewed.proposals, + state.clock.now(), + &lease, + ) + .await + .unwrap() + ); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment + ); + { + #[derive(diesel::QueryableByName)] + struct Base { + #[diesel(sql_type = Text)] + prev_commitment: String, + } + let reserved_proposal = "0xproposed-by-the-signer"; + let rebase = |prev_commitment: String| { + let metadata = metadata.clone(); + async move { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "UPDATE delta_proposals SET prev_commitment = $3 + WHERE account_id = $1 AND commitment = $2", + ) + .bind::(account_id) + .bind::(reserved_proposal) + .bind::(prev_commitment) + .execute(&mut conn) + .await + .unwrap() + } + }; + let original = { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("DELETE FROM execution_submissions WHERE account_id = $1") + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + diesel::sql_query( + "UPDATE execution_reservations SET proposal_id = $2 WHERE account_id = $1", + ) + .bind::(account_id) + .bind::(reserved_proposal) + .execute(&mut conn) + .await + .unwrap(); + diesel::sql_query( + "SELECT prev_commitment FROM delta_proposals + WHERE account_id = $1 AND commitment = $2", + ) + .bind::(account_id) + .bind::(reserved_proposal) + .get_result::(&mut conn) + .await + .unwrap() + .prev_commitment + }; + assert_ne!(original, target); + assert_eq!(current_candidate().await.unsettled_work, None); + + assert_eq!(rebase(target.to_owned()).await, 1); + assert_eq!( + current_candidate().await.unsettled_work.as_deref(), + Some("an execution is reserved") + ); + let current = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + assert!( + !store + .apply( + &eligible, + &reviewed.state.state_json, + &reviewed.auth, + &reviewed.proposals, + state.clock.now(), + ¤t, + ) + .await + .unwrap() + ); + store.leader.release(current).await.unwrap(); + assert_eq!(rebase(original).await, 1); + assert_eq!(current_candidate().await.unsettled_work, None); + } + storage + .submit_delta_proposal( + "0xunreadable-status", + &pending_proposal(8, &other_key.commitment_hex, &[]), + ) + .await + .unwrap(); + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + let updated = diesel::sql_query( + "UPDATE delta_proposals SET status = '{\"status\": \"pending\", \"cosigner_sigs\": 1}' + WHERE account_id = $1 AND commitment = '0xunreadable-status'", + ) + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(updated, 1); + } run_once(&state, &store, MidenNetworkType::Testnet) .await @@ -857,6 +1110,23 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e .collect::>(), ["0xproposed-by-the-signer", "0xsigned-by-the-signer-only"] ); + { + #[derive(diesel::QueryableByName)] + struct Rows { + #[diesel(sql_type = BigInt)] + total: i64, + } + let mut conn = metadata.pool_handle().get().await.unwrap(); + let unreadable = diesel::sql_query( + "SELECT count(*) AS total FROM delta_proposals + WHERE account_id = $1 AND commitment = '0xunreadable-status'", + ) + .bind::(account_id) + .get_result::(&mut conn) + .await + .unwrap(); + assert_eq!(unreadable.total, 1); + } let trimmed = pending_proposal( 5, &signer.commitment_hex, @@ -1001,11 +1271,12 @@ async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_e fn scan_progress(resume_after: &str) -> ScanProgress { ScanProgress { resume_after: resume_after.into(), + ..Default::default() } } #[test] -fn a_scan_completes_only_from_a_whole_pass_that_deferred_no_account() { +fn a_scan_can_complete_across_startups_when_the_pass_deferred_no_account() { let interrupted = ScanSummary { unresolved: 2, last_checked: Some("0xb".into()), @@ -1014,7 +1285,10 @@ fn a_scan_completes_only_from_a_whole_pass_that_deferred_no_account() { }; assert_eq!( interrupted.next_step(scan_progress("0xa")), - ScanStep::Resume(scan_progress("0xb")) + ScanStep::Resume(ScanProgress { + unresolved: 2, + ..scan_progress("0xb") + }) ); let stalled = ScanSummary { @@ -1036,7 +1310,17 @@ fn a_scan_completes_only_from_a_whole_pass_that_deferred_no_account() { ScanStep::Completed { unresolved: 1 } ); assert_eq!( - finished.next_step(scan_progress("0xa")), + finished.next_step(ScanProgress { + unresolved: 2, + ..scan_progress("0xa") + }), + ScanStep::Completed { unresolved: 3 } + ); + assert_eq!( + finished.next_step(ScanProgress { + deferred: true, + ..scan_progress("0xa") + }), ScanStep::Resume(ScanProgress::default()) ); @@ -1291,7 +1575,10 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen store.progress(network).await.unwrap(), ScanProgress::default() ); - let saved = scan_progress(first); + let saved = ScanProgress { + unresolved: 1, + ..scan_progress(first) + }; store.save_progress(network, &saved, &lease).await.unwrap(); assert_eq!(store.progress(network).await.unwrap(), saved); store.leader.release(lease.clone()).await.unwrap(); @@ -1361,17 +1648,6 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen store.progress(network).await.unwrap(), ScanProgress::default() ); - assert!(!store.startup_completed(network).await.unwrap()); - - run_within( - &checking(&[first, second, joined_later]), - &store, - network, - Duration::from_secs(60), - Duration::ZERO, - ) - .await - .unwrap(); assert!(store.startup_completed(network).await.unwrap()); #[derive(diesel::QueryableByName)] @@ -1416,6 +1692,15 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen .await .unwrap() .unwrap(); + store + .record_pending( + &same_nonce, + PendingOutcome::Deferred, + "recorded before the restore", + &lease, + ) + .await + .unwrap(); assert!( store .apply( @@ -1424,7 +1709,11 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen &Auth::MidenEcdsa { cosigner_commitments: vec![Word::from([1, 2, 3, 4_u32]).into_hex()], }, - &ProposalReview::Reviewed(Vec::new()), + &ProposalReview::Reviewed(vec![ReviewedProposal { + commitment: "0xunreadable-status".into(), + status: DeltaStatus::default(), + change: ProposalChange::Keep, + }]), chrono::Utc::now(), &lease, ) @@ -1436,5 +1725,19 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen (restored.commitment.as_str(), restored.nonce), ("0xheld-head", Some(4)) ); + #[derive(diesel::QueryableByName)] + struct PendingRows { + #[diesel(sql_type = BigInt)] + total: i64, + } + let mut conn = metadata.pool_handle().get().await.unwrap(); + let pending = diesel::sql_query( + "SELECT count(*) AS total FROM account_recovery_pending WHERE account_id = $1", + ) + .bind::(joined_later) + .get_result::(&mut conn) + .await + .unwrap(); + assert_eq!(pending.total, 0); store.leader.release(lease).await.unwrap(); } diff --git a/crates/server/src/jobs/account_recovery/proposals.rs b/crates/server/src/jobs/account_recovery/proposals.rs index f980bfa5..b4a2b731 100644 --- a/crates/server/src/jobs/account_recovery/proposals.rs +++ b/crates/server/src/jobs/account_recovery/proposals.rs @@ -4,10 +4,12 @@ use crate::network::AppliedState; use crate::services::proposal_signature::{proposal_tx_summary, verify_proposal_signature}; use crate::storage::ProposalRecord; use crate::storage::postgres::recovery::{ProposalChange, ReviewedProposal}; +use miden_protocol::transaction::TransactionSummary; /// Reviews an account's pending proposals against the signer set of its restored state. A /// proposal none of those signers proposed or approved is removed, and a proposal that stays -/// keeps only their approvals. +/// keeps only their approvals. A proposal whose transaction summary cannot be read stays as +/// it is. pub(super) fn review(restored: &AppliedState, pending: &[ProposalRecord]) -> Vec { pending .iter() @@ -31,7 +33,10 @@ fn change_for(signers: &[ChainCosigner], proposal: &DeltaObject) -> ProposalChan | DeltaStatus::Retained { .. } | DeltaStatus::Discarded { .. } => return ProposalChange::Keep, }; - let approvals = approvals_of(signers, proposal, cosigner_sigs); + let Ok(summary) = proposal_tx_summary(&proposal.delta_payload) else { + return ProposalChange::Keep; + }; + let approvals = approvals_of(signers, &summary, cosigner_sigs); let proposed_by_signer = signers .iter() .any(|signer| signer.commitment.eq_ignore_ascii_case(proposer_id)); @@ -53,16 +58,13 @@ fn change_for(signers: &[ChainCosigner], proposal: &DeltaObject) -> ProposalChan fn approvals_of( signers: &[ChainCosigner], - proposal: &DeltaObject, + summary: &TransactionSummary, signatures: &[CosignerSignature], ) -> Vec { - let Ok(summary) = proposal_tx_summary(&proposal.delta_payload) else { - return Vec::new(); - }; signatures .iter() .filter(|approval| { - verify_proposal_signature(&summary, &approval.signature).is_ok_and(|signer| { + verify_proposal_signature(summary, &approval.signature).is_ok_and(|signer| { signers.iter().any(|known| { known.commitment.eq_ignore_ascii_case(&signer) && known.scheme == Some(approval.signature.scheme()) diff --git a/crates/server/src/schema.rs b/crates/server/src/schema.rs index 3b701b2c..c1e5bda1 100644 --- a/crates/server/src/schema.rs +++ b/crates/server/src/schema.rs @@ -8,6 +8,8 @@ diesel::table! { resume_after -> Varchar, completed_at -> Nullable, unresolved_accounts -> Nullable, + pass_unresolved -> Int8, + pass_deferred -> Bool, } } diff --git a/crates/server/src/storage/postgres_recovery.rs b/crates/server/src/storage/postgres_recovery.rs index 394fcb8b..209c94e4 100644 --- a/crates/server/src/storage/postgres_recovery.rs +++ b/crates/server/src/storage/postgres_recovery.rs @@ -18,16 +18,33 @@ use crate::utils::normalize_commitment_hex; use super::{Pool, lease_fence_is_current, lock_account_metadata}; -const UNSETTLED_ACCOUNT_WORK: &str = " - CASE - WHEN m.has_pending_candidate OR EXISTS ( - SELECT 1 FROM deltas pending WHERE pending.account_id = st.account_id - AND pending.status_kind IN ('candidate', 'retained') - ) THEN 'a delta awaits its on-chain outcome' - WHEN EXISTS ( - SELECT 1 FROM execution_reservations r WHERE r.account_id = st.account_id AND r.released_at IS NULL - ) THEN 'an execution is reserved' - END"; +/// The work in flight that can still land on the account: a delta, or a reserved execution, +/// that starts from `head`, the SQL expression for the account's latest canonical commitment. +fn unsettled_account_work(head: &str) -> String { + format!( + "CASE + WHEN EXISTS ( + SELECT 1 FROM deltas pending WHERE pending.account_id = st.account_id + AND pending.status_kind IN ('candidate', 'retained') + AND pending.prev_commitment = {head} + ) THEN 'a delta awaits its on-chain outcome' + WHEN EXISTS ( + SELECT 1 FROM execution_reservations r + WHERE r.account_id = st.account_id AND r.released_at IS NULL + AND (EXISTS ( + SELECT 1 FROM delta_proposals p + WHERE p.account_id = r.account_id AND p.commitment = r.proposal_id + AND p.prev_commitment = {head} + ) + OR EXISTS ( + SELECT 1 FROM execution_submissions s + WHERE s.account_id = r.account_id AND s.proposal_id = r.proposal_id + AND s.attempt = r.attempt AND s.base_commitment = {head} + )) + ) THEN 'an execution is reserved' + END" + ) +} const CANDIDATE_SOURCE: &str = " FROM account_metadata m @@ -138,6 +155,10 @@ enum ProposalWrite { pub(crate) struct ScanProgress { #[diesel(sql_type = Text)] pub resume_after: String, + #[diesel(sql_type = BigInt)] + pub unresolved: i64, + #[diesel(sql_type = Bool)] + pub deferred: bool, } pub(crate) struct RecoveryStore { @@ -196,7 +217,8 @@ impl RecoveryStore { let network = miden_network_config(network_type)?; let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; diesel::sql_query( - "SELECT resume_after FROM account_recovery_startup WHERE network_config = $1", + "SELECT resume_after, pass_unresolved AS unresolved, pass_deferred AS deferred + FROM account_recovery_startup WHERE network_config = $1", ) .bind::(network) .get_result::(&mut conn) @@ -217,10 +239,13 @@ impl RecoveryStore { .write("record startup recovery progress", |conn| { async move { diesel::sql_query(format!( - "INSERT INTO account_recovery_startup (network_config, resume_after) - SELECT $4, $5 {HELD_LEASE} + "INSERT INTO account_recovery_startup + (network_config, resume_after, pass_unresolved, pass_deferred) + SELECT $4, $5, $6, $7 {HELD_LEASE} ON CONFLICT (network_config) DO UPDATE - SET resume_after = EXCLUDED.resume_after + SET resume_after = EXCLUDED.resume_after, + pass_unresolved = EXCLUDED.pass_unresolved, + pass_deferred = EXCLUDED.pass_deferred WHERE account_recovery_startup.unresolved_accounts IS NULL", )) .bind::(&lease.name) @@ -228,6 +253,8 @@ impl RecoveryStore { .bind::(lease.fence_token) .bind::(network) .bind::(&progress.resume_after) + .bind::(progress.unresolved) + .bind::(progress.deferred) .execute(conn) .await } @@ -254,7 +281,9 @@ impl RecoveryStore { ON CONFLICT (network_config) DO UPDATE SET completed_at = EXCLUDED.completed_at, unresolved_accounts = EXCLUDED.unresolved_accounts, - resume_after = '' + resume_after = '', + pass_unresolved = 0, + pass_deferred = FALSE WHERE account_recovery_startup.unresolved_accounts IS NULL", )) .bind::(&lease.name) @@ -283,7 +312,7 @@ impl RecoveryStore { let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; diesel::sql_query(format!( "SELECT st.account_id, st.nonce AS stored_nonce, st.commitment AS stored_commitment, - d.nonce AS canonical_nonce, ({UNSETTLED_ACCOUNT_WORK}) AS unsettled_work, + d.nonce AS canonical_nonce, ({unsettled_work}) AS unsettled_work, d.new_commitment AS canonical_commitment, st.updated_at AS state_updated_at, m.auth_version, first_held.nonce AS history_starts_at_nonce, @@ -291,6 +320,7 @@ impl RecoveryStore { {CANDIDATE_SOURCE} ORDER BY m.account_id LIMIT $3", + unsettled_work = unsettled_account_work("d.new_commitment"), )) .bind::(after) .bind::(network) @@ -322,6 +352,61 @@ impl RecoveryStore { .map_err(|error| format!("Could not count remaining recovery candidates: {error}")) } + /// Whether a canonical delta of the account starts from or ends at `commitment`. + pub(crate) async fn holds_canonical_commitment( + &self, + account_id: &str, + commitment: &str, + ) -> Result { + self.holds_delta( + account_id, + commitment, + "status_kind = 'canonical' AND (new_commitment = $2 OR prev_commitment = $2)", + ) + .await + } + + /// Whether a delta of the account that still awaits its on-chain outcome ends at + /// `commitment`. + pub(crate) async fn holds_landed_delta( + &self, + account_id: &str, + commitment: &str, + ) -> Result { + self.holds_delta( + account_id, + commitment, + "status_kind IN ('candidate', 'retained') AND new_commitment = $2", + ) + .await + } + + async fn holds_delta( + &self, + account_id: &str, + commitment: &str, + condition: &str, + ) -> Result { + #[derive(diesel::QueryableByName)] + struct Held { + #[diesel(sql_type = Bool)] + held: bool, + } + + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query(format!( + "SELECT EXISTS ( + SELECT 1 FROM deltas WHERE account_id = $1 AND {condition} + ) AS held", + )) + .bind::(account_id) + .bind::(commitment) + .get_result::(&mut conn) + .await + .map(|row| row.held) + .map_err(|error| format!("Could not read the account's deltas: {error}")) + } + pub(crate) async fn record_pending( &self, candidate: &RecoveryCandidate, @@ -413,6 +498,7 @@ impl RecoveryStore { let expected = reviewed.map(|reviewed| { let mut expected: Vec<(String, DeltaStatus)> = reviewed .iter() + .filter(|proposal| proposal.status.is_pending()) .map(|proposal| (proposal.commitment.clone(), proposal.status.clone())) .collect(); expected.sort_by(|left, right| left.0.cmp(&right.0)); @@ -537,6 +623,10 @@ impl RecoveryStore { )) .execute(conn) .await?; + diesel::sql_query("DELETE FROM account_recovery_pending WHERE account_id = $1") + .bind::(account_id) + .execute(conn) + .await?; Ok(true) } .scope_boxed() @@ -608,7 +698,8 @@ impl RecoveryStore { } /// The account's pending proposals and their statuses, locked for the transaction and read -/// the way the storage layer reads them. +/// the way the storage layer reads them. A row whose status cannot be read is left out, as it +/// is from the review this is compared with. async fn pending_proposals( conn: &mut AsyncPgConnection, account_id: &str, @@ -663,13 +754,14 @@ impl RecoveryCandidate { FROM states st JOIN account_metadata m USING (account_id) WHERE st.account_id = $1 AND st.nonce IS NOT DISTINCT FROM $2 AND st.commitment = $3 AND st.updated_at = $4 AND m.auth_version = $5 - AND m.released_at IS NULL AND ({UNSETTLED_ACCOUNT_WORK}) IS NULL + AND m.released_at IS NULL AND ({unsettled_work}) IS NULL AND ($6, $7) = ( SELECT d.nonce, d.new_commitment FROM deltas d WHERE d.account_id = st.account_id AND d.status_kind = 'canonical' ORDER BY d.nonce DESC LIMIT 1 ) FOR UPDATE OF st", + unsettled_work = unsettled_account_work("$7"), )) .bind::(&self.account_id) .bind::, _>(self.stored_nonce) diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index 1782a0e1..7411ceaa 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -381,9 +381,13 @@ accounts that would be restored (`restorable`), deferred and left unresolved. deploy. It passes the mode to the task and lengthens the container health check start period and the load balancer grace period, so the task is not replaced while its listeners are closed. For `apply`, running a single task - (desired count and autoscaling maximum of one) and stopping the running task - before the deploy remain manual steps. Set the variable back to `"off"` - once the check has completed. + (desired count and autoscaling maximum of one) and keeping older tasks from + running during the deploy remain manual steps: turn off the service's + autoscaling, scale the service to zero, wait for its tasks to stop, deploy, + then scale back to one. Stopping a task alone is not enough, because the + service starts a replacement, and autoscaling with a minimum above zero + starts one too. Set the variable back to `"off"` and turn autoscaling back + on once the check has completed. **What it does** @@ -394,10 +398,14 @@ Miden node for the account's current commitment. - If the stored record already matches the network, nothing changes. - If the configured node has no record of the account and Guardian holds no canonical deltas for it, the stored record stands only when it is the - account's first state. Any other record is listed as unresolved, or - deferred while the account has work in flight. -- If Guardian holds the account's canonical deltas from creation (up to 128), - it rebuilds the account from them. When the result matches the network and + account's first state. Any other record is listed as unresolved. +- If the network is past the canonical deltas Guardian holds and no delta in + flight ends at the network's state, the account is listed as unresolved. If + the configured node reports an earlier state of the account, the account is + deferred. +- If Guardian holds the account's canonical deltas from creation, it rebuilds + the account from them. An account whose history takes more than a minute to + rebuild is listed as unresolved. When the result matches the network and is bound to this server's ACK key, Guardian stores it together with its nonce and the account metadata derived from it. The same transaction saves the previous record in `account_recovery_backups`; encrypted state keeps its @@ -407,14 +415,17 @@ Miden node for the account's current commitment. A restore also reviews the account's pending proposals against the signers of the restored state. A proposal that one of those signers proposed or approved -stays, keeping only their approvals. Any other pending proposal is removed. +stays, keeping only their approvals. Any other pending proposal is removed, +except one whose transaction summary cannot be read, which stays as it is. Every proposal a restore changes or removes is saved with the previous record in `account_recovery_backups.prior_proposals`. If the account's proposals cannot be read, the restore leaves them as they are. An account with work in flight (a delta awaiting its on-chain outcome or a -reserved execution) is deferred and checked again on a later startup. A paused -account is checked like any other, and stays paused. +reserved execution) is deferred and checked again on a later startup. Work +counts when it starts from the account's latest canonical state, or when it is +a delta that ends at the network's current state. A paused account is checked +like any other, and stays paused. **How long it takes** @@ -427,9 +438,11 @@ account is checked like any other, and stays paused. - In `apply`, Guardian records where it stopped in `account_recovery_startup.resume_after`, and the next startup in `apply` continues from there. -- The check completes when one startup covers every account, from the first - to the last, without deferring any. Guardian then records how many accounts - it could not bring up to date in +- The check completes when a pass reaches the last account without deferring + any. A pass can span several startups: one that stops early keeps its + position and its counts. A pass that deferred an account is followed by a + new pass from the first account. On completion Guardian records how many + accounts it could not bring up to date in `account_recovery_startup.unresolved_accounts`, and later startups skip the check. Until then, each startup in `apply` runs it again. - An account Guardian holds canonical deltas for, but the configured node has @@ -449,11 +462,23 @@ listeners without finishing the check. After `apply`, `account_recovery_pending` has one row for each account that was checked and not brought up to date. `outcome` is `deferred` (a later startup checks it again) or `unresolved` (Guardian does not hold what it -needs), and `reason` says why. When the canonical history held here does not reach back to the -account's creation, `history_starts_at_nonce` and `history_base_commitment` -show where it begins. An unresolved account the network has a -record of is brought up to date the next time it is registered with its -current state. +needs), and `reason` says why. When the canonical history held here does not +reach back to the account's creation, `history_starts_at_nonce` and +`history_base_commitment` show where it begins. + +An unresolved account the network has a record of is brought up to date the +next time it is registered with its current state. One the network has no +record of cannot be registered again while its stored state remains. Remove +that row with the statement below and keep its output, which is the removed +row. It removes the row only while it still holds the `stored_commitment` +listed for the account in `account_recovery_pending`. The account's next +registration with its first state is then accepted: + +```sql +DELETE FROM states +WHERE account_id = '' AND commitment = '' +RETURNING to_jsonb(states); +``` A row is removed once its account is brought up to date or found unchanged. A row whose `stored_commitment` no longer matches the account's stored state is diff --git a/infra/README.md b/infra/README.md index 3f267405..7998ae34 100644 --- a/infra/README.md +++ b/infra/README.md @@ -116,7 +116,7 @@ server_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/guardian-server # guardian_canonicalization_fast_promotion_enabled = false # guardian_max_pending_candidates_per_account = 4 # opt in to queueing chained candidates (server default 1, max 16; upgrade TS clients first, see docs/CONFIGURATION.md) # guardian_release_sweep_enabled = false # turns the release sweep's walk of the fleet off -# guardian_startup_account_recovery = "dry-run" # then "apply" for one deploy: single task, older task stopped first; see docs/PRODUCTION.md +# guardian_startup_account_recovery = "dry-run" # then "apply" for one deploy: single task, service scaled to zero first; see docs/PRODUCTION.md # guardian_block_watch_enabled = false # kill switch for the committed-block feed # Optional: dashboard operator Falcon public keys managed by Terraform diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index e2a2fad2..e902f607 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -130,7 +130,7 @@ server_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/guardian-server # guardian_canonicalization_fast_promotion_enabled = false # guardian_max_pending_candidates_per_account = 4 # opt in to queueing chained candidates per account (server default 1, max 16) # guardian_release_sweep_enabled = false # turns the release sweep's walk of the fleet off -# guardian_startup_account_recovery = "dry-run" # then "apply" for one deploy: single task, older task stopped first; see docs/PRODUCTION.md +# guardian_startup_account_recovery = "dry-run" # then "apply" for one deploy: single task, service scaled to zero first; see docs/PRODUCTION.md # guardian_release_sweep_rotation_seconds = 21600 # 604800 (weekly) once block watch is healthy # guardian_release_sweep_max_rate_per_second = 5 # guardian_block_watch_enabled = false # kill switch for the committed-block feed (issue #585); unset follows guardian_release_sweep_enabled diff --git a/infra/variables.tf b/infra/variables.tf index abe795c4..35fd1e72 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -579,8 +579,8 @@ variable "guardian_startup_account_recovery" { grace period, so a task is not replaced while the check keeps the listeners closed. For apply the operator arranges the rest: the server skips the check unless it runs as a single task, and the deploy must not - overlap an older task, so stop the running task before the deploy that - turns this on. Set it back to off once the check has completed. + overlap an older task, so turn off autoscaling, scale the service to zero + and wait for its tasks to stop before the deploy that turns this on. Set it back to off once the check has completed. EOT type = string default = "off" From 5df78781fa7d973df74eb29333054c1e6809e50b Mon Sep 17 00:00:00 2001 From: Zeljko Date: Thu, 19 Nov 2026 18:58:46 +0100 Subject: [PATCH 7/8] fix(server): keep a failed streak from deferring a recovery pass - Checks that failed in the streak a scan stops on are made again in the same pass, so they no longer mark the pass as deferred. - Docs: the ECS steps for the check name the Terraform variables that set the task count and autoscaling. --- crates/server/src/jobs/account_recovery.rs | 7 ++++- .../jobs/account_recovery/postgres_tests.rs | 27 +++++++++++++++++++ docs/PRODUCTION.md | 16 ++++++----- infra/variables.tf | 6 +++-- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/crates/server/src/jobs/account_recovery.rs b/crates/server/src/jobs/account_recovery.rs index bf7be759..6b3e4bb4 100644 --- a/crates/server/src/jobs/account_recovery.rs +++ b/crates/server/src/jobs/account_recovery.rs @@ -108,6 +108,9 @@ struct ScanSummary { retryable: usize, unresolved: i64, failed: usize, + /// Failed checks of accounts that the saved position leaves ahead, so the same pass + /// checks them again. + failed_ahead_of_position: usize, last_checked: Option, stopped_early: bool, } @@ -125,7 +128,8 @@ impl ScanSummary { /// from the first account otherwise. fn next_step(&self, started: ScanProgress) -> ScanStep { let unresolved = started.unresolved + self.unresolved; - let deferred = started.deferred || self.retryable > 0 || self.failed > 0; + let deferred = + started.deferred || self.retryable > 0 || self.failed > self.failed_ahead_of_position; if self.stopped_early { return ScanStep::Resume(ScanProgress { resume_after: self.last_checked.clone().unwrap_or(started.resume_after), @@ -467,6 +471,7 @@ async fn run( "Account recovery stopped early after repeated failed checks" ); summary.last_checked = position_before_failed_checks; + summary.failed_ahead_of_position = consecutive_failed_checks; summary.stopped_early = true; break 'scan; } diff --git a/crates/server/src/jobs/account_recovery/postgres_tests.rs b/crates/server/src/jobs/account_recovery/postgres_tests.rs index f09a8b55..6e0a67e1 100644 --- a/crates/server/src/jobs/account_recovery/postgres_tests.rs +++ b/crates/server/src/jobs/account_recovery/postgres_tests.rs @@ -1324,6 +1324,29 @@ fn a_scan_can_complete_across_startups_when_the_pass_deferred_no_account() { ScanStep::Resume(ScanProgress::default()) ); + let rolled_back = ScanSummary { + failed: 3, + failed_ahead_of_position: 3, + last_checked: Some("0xa".into()), + stopped_early: true, + ..Default::default() + }; + assert_eq!( + rolled_back.next_step(ScanProgress::default()), + ScanStep::Resume(scan_progress("0xa")) + ); + let failed_behind_position = ScanSummary { + failed: 4, + ..rolled_back + }; + assert_eq!( + failed_behind_position.next_step(ScanProgress::default()), + ScanStep::Resume(ScanProgress { + deferred: true, + ..scan_progress("0xa") + }) + ); + for deferring in [ ScanSummary { retryable: 1, @@ -1548,6 +1571,10 @@ async fn account_recovery_scan_continues_from_its_saved_position_and_records_pen assert_eq!((outage.checked, outage.failed), (3, 3)); assert!(outage.stopped_early); assert_eq!(outage.last_checked, None); + assert_eq!( + outage.next_step(ScanProgress::default()), + ScanStep::Resume(ScanProgress::default()) + ); let flaky = { let chain = crate::testing::mocks::MockNetworkClient::new() diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index 7411ceaa..46b8c7f5 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -381,13 +381,15 @@ accounts that would be restored (`restorable`), deferred and left unresolved. deploy. It passes the mode to the task and lengthens the container health check start period and the load balancer grace period, so the task is not replaced while its listeners are closed. For `apply`, running a single task - (desired count and autoscaling maximum of one) and keeping older tasks from - running during the deploy remain manual steps: turn off the service's - autoscaling, scale the service to zero, wait for its tasks to stop, deploy, - then scale back to one. Stopping a task alone is not enough, because the - service starts a replacement, and autoscaling with a minimum above zero - starts one too. Set the variable back to `"off"` and turn autoscaling back - on once the check has completed. + and keeping older tasks from running during the deploy remain manual steps. + Terraform sets the task count and autoscaling, so make them through its + variables: first apply `server_autoscaling_enabled = false` with + `server_desired_count = 0` and wait for the tasks to stop, then apply + `guardian_startup_account_recovery = "apply"` with + `server_desired_count = 1`. Stopping a task or scaling the service outside + Terraform is not enough, because the service or the next deploy starts a + replacement. Once the check has completed, set the mode back to `"off"` + and restore the usual task count and autoscaling. **What it does** diff --git a/infra/variables.tf b/infra/variables.tf index 35fd1e72..2aaebe4f 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -579,8 +579,10 @@ variable "guardian_startup_account_recovery" { grace period, so a task is not replaced while the check keeps the listeners closed. For apply the operator arranges the rest: the server skips the check unless it runs as a single task, and the deploy must not - overlap an older task, so turn off autoscaling, scale the service to zero - and wait for its tasks to stop before the deploy that turns this on. Set it back to off once the check has completed. + overlap an older task. Before the deploy that turns this on, apply + server_autoscaling_enabled = false with server_desired_count = 0 and wait + for the tasks to stop, then deploy with server_desired_count = 1. Set it + back to off once the check has completed. EOT type = string default = "off" From 40f4fe4d4ba37be027bd36e9e676ee5595aaff07 Mon Sep 17 00:00:00 2001 From: Zeljko Date: Thu, 19 Nov 2026 19:16:58 +0100 Subject: [PATCH 8/8] chore(server): log nonces on a recovery restore and widen the health check margin - A restore, and its dry run line, log the stored and the restored nonce. - With the startup check on, the container health check gets five retries. - Docs: putting back a record a restore replaced from its backup row, the replica setting for the apply deploy, the dry run's time limit and log fields, and which accounts the check covers. --- crates/server/src/jobs/account_recovery.rs | 4 +++ docs/PRODUCTION.md | 40 ++++++++++++++++++++-- infra/README.md | 2 +- infra/data.tf | 5 +-- infra/ecs.tf | 2 +- 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/crates/server/src/jobs/account_recovery.rs b/crates/server/src/jobs/account_recovery.rs index 6b3e4bb4..75a2cfc4 100644 --- a/crates/server/src/jobs/account_recovery.rs +++ b/crates/server/src/jobs/account_recovery.rs @@ -371,6 +371,8 @@ async fn run( summary.restorable += 1; tracing::info!( account_id = %candidate.account_id, + stored_nonce = ?candidate.stored_nonce, + restored_nonce = ?candidate.canonical_nonce, proposals_changed = restoration.changed_proposals().count(), "Dry run: the account would be restored" ); @@ -393,6 +395,8 @@ async fn run( summary.restored += 1; tracing::info!( account_id = %candidate.account_id, + stored_nonce = ?candidate.stored_nonce, + restored_nonce = ?candidate.canonical_nonce, proposals_changed = restoration.changed_proposals().count(), "Restored chain-verified account state and signers" ); diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index 46b8c7f5..f03c7862 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -367,7 +367,10 @@ progress. It takes no lease, so it runs with any number of replicas, and it checks from the first account each time. It names each proposal a restore would change, and says so when `apply` would not run here (more than one replica, or the check already completed). The final log line counts the -accounts that would be restored (`restorable`), deferred and left unresolved. +accounts that would be restored (`restorable`), deferred (`retryable`) and +left unresolved (`unresolved`). A dry run stops after 5 minutes like `apply`; +that line then shows `stopped_early` and how many accounts it did not reach +(`remaining`). **Applying** @@ -386,7 +389,8 @@ accounts that would be restored (`restorable`), deferred and left unresolved. variables: first apply `server_autoscaling_enabled = false` with `server_desired_count = 0` and wait for the tasks to stop, then apply `guardian_startup_account_recovery = "apply"` with - `server_desired_count = 1`. Stopping a task or scaling the service outside + `server_desired_count = 1`. Leave `guardian_max_replicas` unset or at `1` + for that deploy. Stopping a task or scaling the service outside Terraform is not enough, because the service or the next deploy starts a replacement. Once the check has completed, set the mode back to `"off"` and restore the usual task count and autoscaling. @@ -395,7 +399,9 @@ accounts that would be restored (`restorable`), deferred and left unresolved. The check runs before the API listeners open. For each account whose stored record differs from its latest canonical delta, Guardian asks the configured -Miden node for the account's current commitment. +Miden node for the account's current commitment. It covers accounts recorded +for the configured network and accounts recorded as `local` or `devnet`, which +is how an account registered without a network is stored. - If the stored record already matches the network, nothing changes. - If the configured node has no record of the account and Guardian holds no @@ -493,6 +499,34 @@ JOIN states st ON st.account_id = p.account_id AND st.commitment = p.stored_commitment; ``` +To put back the record a restore replaced, use its row in +`account_recovery_backups`. The first statement changes the state only while +the account still holds the restored commitment: + +```sql +BEGIN; +UPDATE states st +SET state_json = b.prior_state->'state_json', + commitment = b.prior_state->>'commitment', + nonce = (b.prior_state->>'nonce')::bigint, + updated_at = now() +FROM account_recovery_backups b +WHERE b.id = AND st.account_id = b.account_id + AND st.commitment = b.recovered_commitment; +UPDATE account_metadata m SET auth = b.prior_auth, updated_at = now() +FROM account_recovery_backups b +WHERE b.id = AND m.account_id = b.account_id; +INSERT INTO delta_proposals +SELECT p.* FROM account_recovery_backups b, + jsonb_populate_recordset(NULL::delta_proposals, b.prior_proposals) p +WHERE b.id = +ON CONFLICT (account_id, commitment) DO UPDATE + SET status = EXCLUDED.status, status_kind = EXCLUDED.status_kind, + status_timestamp = EXCLUDED.status_timestamp, + delta_payload = EXCLUDED.delta_payload; +COMMIT; +``` + To run the check again after it completed, remove its completion record and restart Guardian: diff --git a/infra/README.md b/infra/README.md index 7998ae34..3eefaac7 100644 --- a/infra/README.md +++ b/infra/README.md @@ -304,7 +304,7 @@ aws ecr delete-repository --repository-name "$ECR_REPO_NAME" --force --region "$ | `guardian_canonicalization_fast_promotion_enabled` | `true` | Enables the recent-candidate promotion-only pass in the ECS task definition | | `guardian_max_pending_candidates_per_account` | server default (`1`) | `GUARDIAN_MAX_PENDING_CANDIDATES_PER_ACCOUNT`: chained candidate deltas one account may hold in flight (issue #17). `1` keeps one in-flight candidate per account; `2` to `16` opts in to queueing back-to-back transactions, after TypeScript clients are upgraded (see [`docs/CONFIGURATION.md`](../docs/CONFIGURATION.md)) | | `guardian_release_sweep_enabled` | `true` | Runs the release sweep's walk of the fleet (the rotation); `false` turns the walk off, and block watch too unless `guardian_block_watch_enabled` is set | -| `guardian_startup_account_recovery` | `off` | `GUARDIAN_STARTUP_ACCOUNT_RECOVERY`: `off`, `dry-run` or `apply`. Checks account records before the listeners open; `dry-run` reports what it would change and writes nothing. Any value but `off` lengthens the container health check start period (to 300 s) and the load balancer grace period (to 420 s) so the task is not replaced during the check. `apply` needs a single task and a deploy with no older task still running; set it back to `off` once the check has completed. See [Startup account recovery](../docs/PRODUCTION.md#startup-account-recovery) | +| `guardian_startup_account_recovery` | `off` | `GUARDIAN_STARTUP_ACCOUNT_RECOVERY`: `off`, `dry-run` or `apply`. Checks account records before the listeners open; `dry-run` reports what it would change and writes nothing. Any value but `off` lengthens the container health check start period (to 300 s, with five retries) and the load balancer grace period (to 420 s) so the task is not replaced during the check. `apply` needs a single task and a deploy with no older task still running; set it back to `off` once the check has completed. See [Startup account recovery](../docs/PRODUCTION.md#startup-account-recovery) | | `guardian_release_sweep_rotation_seconds` | server default (`21600`) | Target time for one release sweep walk of the fleet; `604800` (weekly) is enough once block watch is healthy | | `guardian_block_watch_enabled` | unset (follows `guardian_release_sweep_enabled`) | Runs block watch (issue #585): switch detection from the node's committed-block feed; `false` is its kill switch, `true` runs it even with the release sweep's walk turned off | | `guardian_block_watch_transport` | `auto` | `GUARDIAN_BLOCK_WATCH_TRANSPORT`: `auto` (the block stream, polling when it is unavailable), `stream`, or `poll` | diff --git a/infra/data.tf b/infra/data.tf index 3ba84c87..f16836b1 100644 --- a/infra/data.tf +++ b/infra/data.tf @@ -169,9 +169,10 @@ locals { # Account recovery keeps the listeners closed for about 6 minutes at most: # a 5-minute scan budget, up to 35 seconds waiting for a lease a stopped # task left behind, and the writes in progress when the budget ends. 300 is - # the longest container start period ECS accepts; the health check's three - # retries at 30 seconds cover the remainder. + # the longest container start period ECS accepts; five health check retries + # at 30 seconds cover the remainder with a margin. server_health_check_start_period_seconds = var.guardian_startup_account_recovery == "off" ? 10 : 300 + server_health_check_retries = var.guardian_startup_account_recovery == "off" ? 3 : 5 server_health_check_grace_period_seconds = var.guardian_startup_account_recovery == "off" ? 30 : 420 effective_server_desired_count = var.server_desired_count != null ? var.server_desired_count : (local.is_prod ? 2 : 1) diff --git a/infra/ecs.tf b/infra/ecs.tf index d02cdb74..12c1bbd5 100644 --- a/infra/ecs.tf +++ b/infra/ecs.tf @@ -425,7 +425,7 @@ resource "aws_ecs_task_definition" "server" { command = ["CMD", "/healthcheck"] interval = 30 timeout = 5 - retries = 3 + retries = local.server_health_check_retries startPeriod = local.server_health_check_start_period_seconds }