diff --git a/crates/server/migrations/2026-11-19-000001_account_recovery_backups/down.sql b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/down.sql new file mode 100644 index 00000000..2f06cdd1 --- /dev/null +++ b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/down.sql @@ -0,0 +1 @@ +DROP TABLE account_recovery_backups; diff --git a/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql new file mode 100644 index 00000000..4b2ac994 --- /dev/null +++ b/crates/server/migrations/2026-11-19-000001_account_recovery_backups/up.sql @@ -0,0 +1,9 @@ +CREATE TABLE account_recovery_backups ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + account_id VARCHAR(128) NOT NULL, + prior_state JSONB NOT NULL, + prior_auth JSONB NOT NULL, + prior_proposals JSONB NOT NULL DEFAULT '[]'::jsonb, + recovered_commitment VARCHAR(128) NOT NULL, + recovered_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/crates/server/migrations/2026-11-19-000002_account_recovery_startup/down.sql b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/down.sql new file mode 100644 index 00000000..4b4d93cc --- /dev/null +++ b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/down.sql @@ -0,0 +1 @@ +DROP TABLE account_recovery_startup; diff --git a/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql new file mode 100644 index 00000000..e52834fa --- /dev/null +++ b/crates/server/migrations/2026-11-19-000002_account_recovery_startup/up.sql @@ -0,0 +1,8 @@ +CREATE TABLE account_recovery_startup ( + network_config JSONB PRIMARY KEY, + resume_after VARCHAR(128) NOT NULL DEFAULT '', + completed_at TIMESTAMPTZ, + unresolved_accounts BIGINT, + pass_unresolved BIGINT NOT NULL DEFAULT 0, + pass_deferred BOOLEAN NOT NULL DEFAULT FALSE +); diff --git a/crates/server/migrations/2026-11-19-000003_account_recovery_pending/down.sql b/crates/server/migrations/2026-11-19-000003_account_recovery_pending/down.sql new file mode 100644 index 00000000..bdb06264 --- /dev/null +++ b/crates/server/migrations/2026-11-19-000003_account_recovery_pending/down.sql @@ -0,0 +1 @@ +DROP TABLE account_recovery_pending; diff --git a/crates/server/migrations/2026-11-19-000003_account_recovery_pending/up.sql b/crates/server/migrations/2026-11-19-000003_account_recovery_pending/up.sql new file mode 100644 index 00000000..426c0ffe --- /dev/null +++ b/crates/server/migrations/2026-11-19-000003_account_recovery_pending/up.sql @@ -0,0 +1,11 @@ +CREATE TABLE account_recovery_pending ( + account_id VARCHAR(128) PRIMARY KEY, + outcome TEXT NOT NULL CHECK (outcome IN ('deferred', 'unresolved')), + reason TEXT NOT NULL, + stored_commitment VARCHAR(128) NOT NULL, + canonical_nonce BIGINT, + canonical_commitment VARCHAR(128), + history_starts_at_nonce BIGINT, + history_base_commitment VARCHAR(128), + recorded_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/crates/server/src/builder/handle.rs b/crates/server/src/builder/handle.rs index a486739e..dadd7888 100644 --- a/crates/server/src/builder/handle.rs +++ b/crates/server/src/builder/handle.rs @@ -54,6 +54,11 @@ pub struct ServerHandle { pub(crate) stats_leader: std::sync::Arc, /// Single-owner lease for the chain-driven release sweep (issue #434). pub(crate) release_sweep_leader: std::sync::Arc, + #[cfg(feature = "postgres")] + pub(crate) account_recovery: Option<( + std::sync::Arc, + crate::metadata::MidenNetworkType, + )>, pub(crate) startup_info: StartupInfo, pub(crate) cors_layer: Option, pub(crate) rate_limit_config: Option, @@ -65,7 +70,89 @@ pub struct ServerHandle { pub(crate) grpc_port: u16, } +/// Whether the server checks its account records at startup, and whether that check writes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StartupRecoveryMode { + Off, + DryRun, + Apply, +} + +impl std::str::FromStr for StartupRecoveryMode { + type Err = String; + + fn from_str(value: &str) -> Result { + match value.trim() { + "off" => Ok(Self::Off), + "dry-run" => Ok(Self::DryRun), + "apply" => Ok(Self::Apply), + other => Err(format!("must be off, dry-run or apply, got {other:?}")), + } + } +} + +#[cfg(feature = "postgres")] +#[derive(Debug, thiserror::Error)] +pub enum StartupRecoveryError { + #[error("Another process already holds the account recovery lease")] + LeaseBusy, + #[error("Could not establish exclusive account recovery: {0}")] + LeaseUnavailable(String), + #[error("{0}")] + Incomplete(String), +} + +#[cfg(feature = "postgres")] +impl From for StartupRecoveryError { + fn from(reason: String) -> Self { + Self::Incomplete(reason) + } +} + impl ServerHandle { + /// Recover accounts once before a stop-before-start deployment opens its listeners. + #[cfg(feature = "postgres")] + pub async fn recover_accounts_on_startup( + &self, + mode: StartupRecoveryMode, + ) -> Result<(), StartupRecoveryError> { + if mode == StartupRecoveryMode::Off { + return Ok(()); + } + let (store, network_type) = self + .account_recovery + .as_ref() + .ok_or_else(|| "Account recovery requires shared Postgres storage".to_owned())?; + let single_replica = crate::middleware::rate_limit::max_replicas_from_env()? == 1; + let canonicalizes = self.app_state.canonicalization.is_some(); + match mode { + StartupRecoveryMode::Off => Ok(()), + StartupRecoveryMode::DryRun => { + if !single_replica { + tracing::warn!( + "Dry run: apply would be skipped because GUARDIAN_MAX_REPLICAS exceeds 1" + ); + } + if !canonicalizes { + tracing::warn!("Dry run: apply would not run without canonicalization"); + } + crate::jobs::account_recovery::dry_run(&self.app_state, store, *network_type).await + } + StartupRecoveryMode::Apply if !single_replica => { + tracing::warn!( + "Skipping startup account recovery because GUARDIAN_MAX_REPLICAS exceeds 1" + ); + Ok(()) + } + StartupRecoveryMode::Apply if !canonicalizes => Err(StartupRecoveryError::Incomplete( + "Account recovery requires canonicalization".into(), + )), + StartupRecoveryMode::Apply => { + crate::jobs::account_recovery::run_once(&self.app_state, store, *network_type).await + } + } + } + /// Run the server with the configured settings pub async fn run(self) { self.startup_info.log(); @@ -821,3 +908,18 @@ fn start_session_sweep_worker(state: AppState) { } }); } + +#[cfg(test)] +mod startup_recovery_mode_tests { + use super::StartupRecoveryMode; + + #[test] + fn the_mode_has_three_values_and_takes_no_boolean() { + assert_eq!("off".parse(), Ok(StartupRecoveryMode::Off)); + assert_eq!(" dry-run ".parse(), Ok(StartupRecoveryMode::DryRun)); + assert_eq!("apply".parse(), Ok(StartupRecoveryMode::Apply)); + for rejected in ["true", "false", "", "Apply"] { + assert!(rejected.parse::().is_err()); + } + } +} diff --git a/crates/server/src/builder/mod.rs b/crates/server/src/builder/mod.rs index 4891c869..9a4c67d4 100644 --- a/crates/server/src/builder/mod.rs +++ b/crates/server/src/builder/mod.rs @@ -506,6 +506,11 @@ impl ServerBuilder { let ack = self.ack.ok_or("AckRegistry not set. Use .ack(...)")?; let coordination = self.coordination; + #[cfg(feature = "postgres")] + let account_recovery = coordination + .as_ref() + .and_then(|handles| handles.account_recovery.clone()) + .map(|store| (store, crate::metadata::MidenNetworkType::from(network_type))); // Fail closed before anything else: the Postgres backend must never fall // back to per-process coordination (AlwaysLeader + in-memory sessions), // which would let every replica run canonicalization and split auth @@ -790,6 +795,8 @@ impl ServerBuilder { leader, stats_leader, release_sweep_leader, + #[cfg(feature = "postgres")] + account_recovery, startup_info, cors_layer: self.cors_layer, rate_limit_config: Some(rate_limit_config), diff --git a/crates/server/src/coordination/mod.rs b/crates/server/src/coordination/mod.rs index 236a7dff..b4311f5a 100644 --- a/crates/server/src/coordination/mod.rs +++ b/crates/server/src/coordination/mod.rs @@ -72,6 +72,8 @@ pub struct CoordinationHandles { pub stats_leader: Arc, /// Single-owner lease for the chain-driven release sweep (issue #434). pub release_sweep_leader: Arc, + #[cfg(feature = "postgres")] + pub(crate) account_recovery: Option>, /// Account-scoped leases that fence Guardian executions (issue #254). pub execution_leases: Arc, /// Shared publication store for the `/dashboard/stats` aggregate. @@ -112,6 +114,8 @@ impl CoordinationHandles { RELEASE_SWEEP_LEASE, "single-process", )), + #[cfg(feature = "postgres")] + account_recovery: None, execution_leases: Arc::new(InMemoryExecutionLeases::new()), stats_store: Arc::new(InMemoryStatsStore::new()), block_watch_store: Arc::new(InMemoryBlockWatchStore::new()), @@ -155,7 +159,14 @@ impl CoordinationHandles { release_sweep_leader: Arc::new(PgLeaseElector::new( pool.clone(), RELEASE_SWEEP_LEASE, - holder_id, + holder_id.clone(), + )), + account_recovery: Some(Arc::new( + crate::storage::postgres::recovery::RecoveryStore::new( + pool.clone(), + cipher.clone(), + holder_id, + ), )), execution_leases: Arc::new(PgExecutionLeases::new(pool.clone())), stats_store: Arc::new(PgStatsStore::new(pool.clone(), cipher)), diff --git a/crates/server/src/jobs/account_recovery.rs b/crates/server/src/jobs/account_recovery.rs new file mode 100644 index 00000000..75a2cfc4 --- /dev/null +++ b/crates/server/src/jobs/account_recovery.rs @@ -0,0 +1,810 @@ +use std::{fmt::Display, future::Future, time::Duration}; + +use guardian_shared::SignatureScheme; +use tokio::time::{Instant, timeout, timeout_at}; +use tokio_util::sync::CancellationToken; + +use crate::builder::handle::StartupRecoveryError; +use crate::coordination::Lease; +use crate::delta_object::DeltaObject; +use crate::jobs::canonicalization::spawn_lease_renewal; +use crate::metadata::{Auth, MidenNetworkType}; +use crate::network::{ + AppliedState, NetworkClient, ReconstructError, RpcReadMode, StateVerification, reconstructor, +}; +use crate::state::AppState; +use crate::storage::postgres::recovery::{ + PendingOutcome, ProposalChange, ProposalReview, RecoveryCandidate, RecoveryStore, + ReviewedProposal, ScanProgress, +}; +use crate::storage::{AccountDeltaCursor, is_record_corruption}; + +const PAGE_SIZE: i64 = 100; +const HISTORY_PAGE: u32 = 128; +const REBUILD_BUDGET: Duration = Duration::from_secs(60); +const LEASE_TTL: Duration = Duration::from_secs(30); +const SCAN_BUDGET: Duration = Duration::from_secs(300); +const REMAINING_COUNT_BUDGET: Duration = Duration::from_secs(1); +const SCAN_CONTROL_BUDGET: Duration = Duration::from_secs(6); +const STALE_LEASE_WAIT: Duration = Duration::from_secs(35); +const LEASE_RETRY_INTERVAL: Duration = Duration::from_secs(2); +const MAX_CONSECUTIVE_FAILED_CHECKS: usize = 3; + +/// An account rebuilt from its canonical history, or the reason that history does not +/// rebuild it. +#[derive(Debug)] +enum Rebuild { + Rebuilt(AppliedState), + Unresolved(String), +} + +#[derive(Debug)] +enum RecoveryOutcome { + Unchanged, + Verified(Box), + Retryable(String), + Unresolved(String), +} + +/// What a restore writes for one account. +#[derive(Debug)] +struct Restoration { + state: AppliedState, + auth: Auth, + proposals: ProposalReview, +} + +impl Restoration { + /// The proposals a restore removes or rewrites. + fn changed_proposals(&self) -> impl Iterator { + let reviewed = match &self.proposals { + ProposalReview::Reviewed(reviewed) => reviewed.as_slice(), + ProposalReview::Unreadable => &[], + }; + reviewed + .iter() + .filter(|proposal| proposal.change != ProposalChange::Keep) + } + + fn log_proposal_changes(&self, account_id: &str, dry_run: bool) { + if self.proposals == ProposalReview::Unreadable { + tracing::warn!( + %account_id, + dry_run, + "Account recovery could not read the pending proposals and leaves them as they are" + ); + } + for proposal in self.changed_proposals() { + tracing::info!( + %account_id, + proposal = %proposal.commitment, + change = proposal.change.label(), + dry_run, + "Account recovery proposal review" + ); + } + } +} + +enum PendingRecord { + Clear, + ClearedByRestore, + Write(PendingOutcome, String), +} + +/// Whether a scan writes what it finds. A dry run reads and reports only. +#[derive(Clone, Copy)] +enum ScanWrites<'a> { + Apply(&'a Lease), + DryRun, +} + +#[derive(Debug, Default)] +struct ScanSummary { + checked: usize, + restored: usize, + restorable: usize, + unchanged: usize, + retryable: usize, + unresolved: i64, + failed: usize, + /// Failed checks of accounts that the saved position leaves ahead, so the same pass + /// checks them again. + failed_ahead_of_position: usize, + last_checked: Option, + stopped_early: bool, +} + +#[derive(Debug, PartialEq, Eq)] +enum ScanStep { + Completed { unresolved: i64 }, + Resume(ScanProgress), +} + +impl ScanSummary { + /// What a scan that started at `started` leaves for the next startup. A scan that stops + /// early keeps its position and the counts of its pass. A pass that reaches the last + /// account completes the scan when it deferred no account, and is followed by a new pass + /// from the first account otherwise. + fn next_step(&self, started: ScanProgress) -> ScanStep { + let unresolved = started.unresolved + self.unresolved; + let deferred = + started.deferred || self.retryable > 0 || self.failed > self.failed_ahead_of_position; + if self.stopped_early { + return ScanStep::Resume(ScanProgress { + resume_after: self.last_checked.clone().unwrap_or(started.resume_after), + unresolved, + deferred, + }); + } + if deferred { + ScanStep::Resume(ScanProgress::default()) + } else { + ScanStep::Completed { unresolved } + } + } +} + +pub(crate) async fn run_once( + state: &AppState, + store: &RecoveryStore, + network_type: MidenNetworkType, +) -> Result<(), StartupRecoveryError> { + run_within(state, store, network_type, SCAN_BUDGET, STALE_LEASE_WAIT).await +} + +/// Takes the recovery lease, waiting out one a stopped process left behind. A lease that is +/// still held after `wait` belongs to a live process. +async fn acquire_lease( + store: &RecoveryStore, + wait: Duration, +) -> Result { + let give_up_at = Instant::now() + wait; + let mut waiting = false; + loop { + let acquired = bounded_db_call( + "acquire the recovery lease", + store.leader.try_acquire(LEASE_TTL), + ) + .await + .map_err(StartupRecoveryError::LeaseUnavailable)?; + match acquired { + Some(lease) => return Ok(lease), + None if Instant::now() + LEASE_RETRY_INTERVAL > give_up_at => { + return Err(StartupRecoveryError::LeaseBusy); + } + None => { + if !waiting { + tracing::info!("Waiting for the account recovery lease another process holds"); + waiting = true; + } + tokio::time::sleep(LEASE_RETRY_INTERVAL).await + } + } + } +} + +/// Runs the scan without a lease and without writing anything: the log lines say what an +/// applying run would do with each account. +pub(crate) async fn dry_run( + state: &AppState, + store: &RecoveryStore, + network_type: MidenNetworkType, +) -> Result<(), StartupRecoveryError> { + if bounded_db_call( + "read the recovery checkpoint", + store.startup_completed(network_type), + ) + .await + .unwrap_or(false) + { + tracing::info!( + ?network_type, + "Dry run: account recovery already completed, so apply would skip the scan" + ); + } + run( + state, + store, + network_type, + ScanWrites::DryRun, + &CancellationToken::new(), + "", + Instant::now() + SCAN_BUDGET, + ) + .await + .map(drop) + .map_err(StartupRecoveryError::Incomplete) +} + +async fn run_within( + state: &AppState, + store: &RecoveryStore, + network_type: MidenNetworkType, + budget: Duration, + stale_lease_wait: Duration, +) -> Result<(), StartupRecoveryError> { + let lease = acquire_lease(store, stale_lease_wait).await?; + let cancel = CancellationToken::new(); + let renewal = spawn_lease_renewal( + store.leader.clone(), + lease.clone(), + LEASE_TTL, + LEASE_TTL / 3, + cancel.clone(), + "Account recovery", + ); + let result = async { + if bounded_db_call( + "read the recovery checkpoint", + store.startup_completed(network_type), + ) + .await? + { + tracing::info!( + ?network_type, + "Account recovery already completed; skipping the scan" + ); + return Ok(()); + } + let started = + bounded_db_call("read recovery progress", store.progress(network_type)).await?; + let summary = run( + state, + store, + network_type, + ScanWrites::Apply(&lease), + &cancel, + &started.resume_after, + Instant::now() + budget, + ) + .await?; + match summary.next_step(started) { + ScanStep::Completed { unresolved } => { + bounded_db_call( + "record recovery completion", + store.complete_startup(network_type, unresolved, &lease), + ) + .await?; + tracing::info!( + unresolved, + "Account recovery completed; later startups skip the scan" + ); + Ok(()) + } + ScanStep::Resume(progress) => { + tracing::warn!( + resume_after = %progress.resume_after, + "Account recovery will continue on the next eligible startup" + ); + bounded_db_call( + "save recovery progress", + store.save_progress(network_type, &progress, &lease), + ) + .await + } + } + } + .await; + let held = bounded_db_call( + "confirm the recovery lease", + store.leader.verify_held(&lease), + ) + .await; + renewal.abort(); + let _ = renewal.await; + let released = bounded_db_call("release the recovery lease", store.leader.release(lease)).await; + if !held.map_err(StartupRecoveryError::LeaseUnavailable)? { + return Err(StartupRecoveryError::LeaseUnavailable( + "The recovery lease was lost".into(), + )); + } + result + .and(released) + .map_err(StartupRecoveryError::Incomplete) +} + +async fn bounded_db_call( + operation: &str, + work: impl Future>, +) -> Result { + timeout(SCAN_CONTROL_BUDGET, work) + .await + .map_err(|_| format!("Timed out while trying to {operation}"))? + .map_err(|error| error.to_string()) +} + +async fn run( + state: &AppState, + store: &RecoveryStore, + network_type: MidenNetworkType, + writes: ScanWrites<'_>, + cancel: &CancellationToken, + resume_after: &str, + deadline: Instant, +) -> Result { + let dry_run = matches!(writes, ScanWrites::DryRun); + tracing::info!(?network_type, %resume_after, dry_run, "Starting account recovery scan"); + let mut after = resume_after.to_owned(); + let mut summary = ScanSummary::default(); + let mut consecutive_failed_checks = 0; + let mut position_before_failed_checks = None; + 'scan: loop { + let candidates = + match timeout_at(deadline, store.candidates(network_type, &after, PAGE_SIZE)).await { + Ok(result) => result?, + Err(_) => { + summary.stopped_early = true; + break; + } + }; + let Some(last) = candidates.last() else { break }; + after.clone_from(&last.account_id); + + for candidate in candidates { + if cancel.is_cancelled() { + return Err("account recovery lease lost".into()); + } + let result = match timeout_at(deadline, check_candidate(state, store, &candidate)).await + { + Ok(result) => result, + Err(_) => { + summary.stopped_early = true; + break 'scan; + } + }; + if cancel.is_cancelled() { + return Err("account recovery lease lost".into()); + } + summary.checked += 1; + consecutive_failed_checks = match &result { + Ok(_) => 0, + Err(_) => consecutive_failed_checks + 1, + }; + let record = match result { + Ok(RecoveryOutcome::Unchanged) => { + summary.unchanged += 1; + PendingRecord::Clear + } + Ok(RecoveryOutcome::Verified(restoration)) => match writes { + ScanWrites::DryRun => { + summary.restorable += 1; + tracing::info!( + account_id = %candidate.account_id, + stored_nonce = ?candidate.stored_nonce, + restored_nonce = ?candidate.canonical_nonce, + proposals_changed = restoration.changed_proposals().count(), + "Dry run: the account would be restored" + ); + restoration.log_proposal_changes(&candidate.account_id, true); + PendingRecord::Clear + } + ScanWrites::Apply(lease) => { + match store + .apply( + &candidate, + &restoration.state.state_json, + &restoration.auth, + &restoration.proposals, + state.clock.now(), + lease, + ) + .await + { + Ok(true) => { + summary.restored += 1; + tracing::info!( + account_id = %candidate.account_id, + stored_nonce = ?candidate.stored_nonce, + restored_nonce = ?candidate.canonical_nonce, + proposals_changed = restoration.changed_proposals().count(), + "Restored chain-verified account state and signers" + ); + restoration.log_proposal_changes(&candidate.account_id, false); + PendingRecord::ClearedByRestore + } + Ok(false) => { + summary.retryable += 1; + tracing::info!(account_id = %candidate.account_id, "Account became ineligible or changed during recovery"); + PendingRecord::Write( + PendingOutcome::Deferred, + "the account changed or became ineligible during recovery" + .into(), + ) + } + Err(reason) => { + summary.failed += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery transaction failed"); + PendingRecord::Write( + PendingOutcome::Deferred, + "restore transaction failed; see Guardian logs".into(), + ) + } + } + } + }, + Ok(RecoveryOutcome::Retryable(reason)) => { + summary.retryable += 1; + tracing::info!(account_id = %candidate.account_id, %reason, dry_run, "Account recovery deferred until a later startup"); + PendingRecord::Write(PendingOutcome::Deferred, reason) + } + Ok(RecoveryOutcome::Unresolved(reason)) => { + summary.unresolved += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, dry_run, "Account recovery needs additional evidence"); + PendingRecord::Write(PendingOutcome::Unresolved, reason) + } + Err(reason) => { + summary.failed += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, dry_run, "Account recovery could not check the candidate"); + PendingRecord::Write( + PendingOutcome::Deferred, + "account verification failed; see Guardian logs".into(), + ) + } + }; + if let ScanWrites::Apply(lease) = writes { + let recorded = match record { + PendingRecord::ClearedByRestore => Ok(()), + PendingRecord::Clear => { + bounded_db_call( + "clear a pending recovery record", + store.clear_pending(&candidate.account_id, lease), + ) + .await + } + PendingRecord::Write(outcome, reason) => { + bounded_db_call( + "write a pending recovery record", + store.record_pending(&candidate, outcome, &reason, lease), + ) + .await + } + }; + if let Err(reason) = recorded { + summary.failed += 1; + tracing::warn!(account_id = %candidate.account_id, %reason, "Account recovery could not update the pending record"); + } + } + summary.last_checked = Some(candidate.account_id); + if consecutive_failed_checks == 0 { + position_before_failed_checks.clone_from(&summary.last_checked); + } + if consecutive_failed_checks >= MAX_CONSECUTIVE_FAILED_CHECKS { + tracing::warn!( + consecutive_failed_checks, + "Account recovery stopped early after repeated failed checks" + ); + summary.last_checked = position_before_failed_checks; + summary.failed_ahead_of_position = consecutive_failed_checks; + summary.stopped_early = true; + break 'scan; + } + if Instant::now() >= deadline { + summary.stopped_early = true; + break 'scan; + } + } + } + let remaining = if summary.stopped_early { + let position = summary.last_checked.as_deref().unwrap_or(resume_after); + timeout( + REMAINING_COUNT_BUDGET, + store.remaining_candidates(network_type, position), + ) + .await + .ok() + .and_then(Result::ok) + } else { + Some(0) + }; + tracing::info!( + dry_run, + checked = summary.checked, + restored = summary.restored, + restorable = summary.restorable, + unchanged = summary.unchanged, + retryable = summary.retryable, + unresolved = summary.unresolved, + failed = summary.failed, + stopped_early = summary.stopped_early, + ?remaining, + "Finished account recovery scan" + ); + Ok(summary) +} + +async fn check_candidate( + state: &AppState, + store: &RecoveryStore, + candidate: &RecoveryCandidate, +) -> Result { + let chain = state + .network_client + .verify_commitment( + &candidate.account_id, + &candidate.stored_commitment, + RpcReadMode::Configured, + ) + .await?; + if matches!(chain, StateVerification::Match) { + return Ok(RecoveryOutcome::Unchanged); + } + if matches!(chain, StateVerification::Absent) { + if candidate.canonical_nonce.is_some() { + return Ok(RecoveryOutcome::Retryable( + "the configured node has no record of the account".into(), + )); + } + if holds_first_state(state, candidate).await? { + return Ok(RecoveryOutcome::Unchanged); + } + } + if let Some(unsettled_work) = &candidate.unsettled_work { + return Ok(RecoveryOutcome::Retryable(unsettled_work.clone())); + } + if let StateVerification::Mismatch { on_chain } = &chain + && candidate.canonical_commitment.as_deref() != Some(on_chain.as_str()) + && store + .holds_landed_delta(&candidate.account_id, on_chain) + .await? + { + return Ok(RecoveryOutcome::Retryable( + "a delta that landed awaits its promotion".into(), + )); + } + + let (Some(canonical_nonce), Some(target)) = ( + candidate.canonical_nonce, + candidate.canonical_commitment.as_deref(), + ) else { + let reason = match chain { + StateVerification::Absent => { + "the configured node has no record of the account and no canonical history is held" + } + StateVerification::Match | StateVerification::Mismatch { .. } => { + "no canonical history supplies a recovery target" + } + }; + return Ok(RecoveryOutcome::Unresolved(reason.into())); + }; + match &chain { + StateVerification::Mismatch { on_chain } if on_chain == target => {} + StateVerification::Mismatch { on_chain } => { + let earlier = store + .holds_canonical_commitment(&candidate.account_id, on_chain) + .await?; + return Ok(if earlier { + RecoveryOutcome::Retryable( + "the configured node reports an earlier state of the account".into(), + ) + } else { + RecoveryOutcome::Unresolved( + "the configured node's commitment is missing from the canonical history held here".into(), + ) + }); + } + StateVerification::Match | StateVerification::Absent => { + return Ok(RecoveryOutcome::Retryable( + "latest canonical delta does not match the current chain head".into(), + )); + } + } + let rebuild = rebuild(state, &candidate.account_id, canonical_nonce, HISTORY_PAGE); + let rebuilt = match timeout(REBUILD_BUDGET, rebuild).await { + Ok(rebuilt) => match rebuilt? { + Rebuild::Rebuilt(rebuilt) => rebuilt, + Rebuild::Unresolved(reason) => return Ok(RecoveryOutcome::Unresolved(reason)), + }, + Err(_) => { + return Ok(RecoveryOutcome::Unresolved( + "the canonical history takes too long to rebuild".into(), + )); + } + }; + + let network = state.network_client.clone(); + let account_id = candidate.account_id.clone(); + let target = target.to_owned(); + let proof = reconstructor() + .run_background(move || prove_recovery(network.as_ref(), &account_id, &target, rebuilt)) + .await; + let (applied, auth) = match proof { + Ok(proof) => proof, + Err(ReconstructError::Operation(reason)) => return Ok(RecoveryOutcome::Unresolved(reason)), + Err(ReconstructError::Task(reason)) => { + return Err(format!("state reconstruction failed: {reason}")); + } + }; + + let scheme = auth.guardian_scheme(); + if applied.guardian_commitment.as_deref() != Some(state.ack.commitment(&scheme).as_str()) { + return Ok(RecoveryOutcome::Unresolved( + "recovered state is bound to a different Guardian key".into(), + )); + } + + let proposals = match state + .storage + .pull_pending_proposals(&candidate.account_id) + .await + { + Ok(pending) => ProposalReview::Reviewed(proposals::review(&applied, &pending)), + Err(reason) if is_record_corruption(&reason) => ProposalReview::Unreadable, + Err(reason) => return Err(reason), + }; + + match state + .network_client + .verify_commitment( + &candidate.account_id, + &applied.commitment, + RpcReadMode::Configured, + ) + .await? + { + StateVerification::Match => {} + StateVerification::Mismatch { .. } | StateVerification::Absent => { + return Ok(RecoveryOutcome::Retryable( + "reconstructed state does not match the current chain head".into(), + )); + } + } + + Ok(RecoveryOutcome::Verified(Box::new(Restoration { + state: applied, + auth, + proposals, + }))) +} + +/// Whether the stored state is a first state, which stands without a record on the network. +async fn holds_first_state( + state: &AppState, + candidate: &RecoveryCandidate, +) -> Result { + if candidate.stored_nonce.is_some_and(|nonce| nonce != 0) { + return Ok(false); + } + match state.storage.pull_state(&candidate.account_id).await { + Ok(stored) => Ok(state + .network_client + .validate_first_state(&stored.state_json) + .is_ok()), + Err(reason) if is_record_corruption(&reason) => Ok(false), + Err(reason) => Err(reason), + } +} + +/// Rebuilds the account from its canonical deltas, read `page` at a time from the first one. +async fn rebuild( + state: &AppState, + account_id: &str, + canonical_nonce: i64, + page: u32, +) -> Result { + let mut rebuilt: Option = None; + let mut first = 1_i64; + let Some(end) = canonical_nonce.checked_add(1) else { + return Ok(Rebuild::Unresolved( + "canonical nonce exceeds the storage range".into(), + )); + }; + while first < end { + let after = first.saturating_add(i64::from(page)).min(end); + let expected = u32::try_from(after - first).map_err(|error| error.to_string())?; + let read = state + .storage + .list_canonical_deltas_paged( + account_id, + expected, + Some(AccountDeltaCursor { last_nonce: after }), + ) + .await; + let deltas = match read { + Ok(deltas) => deltas, + Err(reason) if is_record_corruption(&reason) => { + return Ok(Rebuild::Unresolved(reason)); + } + Err(reason) => return Err(reason), + }; + if u32::try_from(deltas.len()) != Ok(expected) { + return Ok(Rebuild::Unresolved( + "missing canonical deltas from account creation".into(), + )); + } + let network = state.network_client.clone(); + let account = account_id.to_owned(); + let replayed = reconstructor() + .run_background(move || replay(network.as_ref(), &account, rebuilt, deltas)) + .await; + rebuilt = match replayed { + Ok(applied) => Some(applied), + Err(ReconstructError::Operation(reason)) => return Ok(Rebuild::Unresolved(reason)), + Err(ReconstructError::Task(reason)) => { + return Err(format!("state reconstruction failed: {reason}")); + } + }; + first = after; + } + Ok(match rebuilt { + Some(rebuilt) => Rebuild::Rebuilt(rebuilt), + None => Rebuild::Unresolved("canonical history is empty".into()), + }) +} + +/// The rebuilt state with the signer set it carries, once it is the expected account head. +fn prove_recovery( + network: &dyn NetworkClient, + account_id: &str, + target: &str, + applied: AppliedState, +) -> Result<(AppliedState, Auth), String> { + let head = network.get_state_head(account_id, &applied.state_json)?; + if head.commitment != target || head.nonce != applied.nonce { + return Err("reconstructed state does not match the expected account head".into()); + } + let binding = network + .account_auth_binding(&applied.state_json)? + .ok_or("missing account auth binding")?; + network.validate_multisig_auth(&applied.state_json)?; + let scheme = binding + .guardian_scheme + .ok_or("missing on-chain Guardian signature scheme")?; + let auth = recovered_auth(scheme, &applied)?; + Ok((applied, auth)) +} + +/// Applies canonical deltas, newest first as storage returns them, on top of the state rebuilt +/// so far. Without one, the first delta creates the account. +fn replay( + network: &dyn NetworkClient, + account_id: &str, + rebuilt: Option, + deltas: Vec, +) -> Result { + let first_nonce = match &rebuilt { + Some(rebuilt) => rebuilt.nonce.ok_or("rebuilt state has no nonce")? + 1, + None => 1, + }; + let mut reconstructed = rebuilt; + for (nonce, delta) in (first_nonce..).zip(deltas.into_iter().rev()) { + if delta.account_id != account_id || delta.nonce != nonce || !delta.status.is_canonical() { + return Err("canonical history is not contiguous from account creation".into()); + } + let applied = match reconstructed { + None => network.apply_delta(&serde_json::Value::Null, &delta.delta_payload)?, + Some(previous) => network.verify_and_apply_delta( + &delta.prev_commitment, + &previous.state_json, + &delta.delta_payload, + )?, + }; + if delta.new_commitment.as_deref() != Some(applied.commitment.as_str()) + || applied.nonce != Some(delta.nonce) + { + return Err("reconstructed delta does not match its canonical row".into()); + } + reconstructed = Some(applied); + } + reconstructed.ok_or_else(|| "canonical history is empty".into()) +} + +fn recovered_auth(scheme: SignatureScheme, state: &AppliedState) -> Result { + if state.cosigners.is_empty() || state.cosigners.iter().any(|signer| signer.scheme.is_none()) { + return Err("recovered signer set is empty or has an unknown signature scheme".into()); + } + let baseline = match scheme { + SignatureScheme::Falcon => Auth::MidenFalconRpo { + cosigner_commitments: Vec::new(), + }, + SignatureScheme::Ecdsa => Auth::MidenEcdsa { + cosigner_commitments: Vec::new(), + }, + }; + Ok(baseline.with_chain_cosigners(state.cosigners.clone())) +} + +mod proposals; + +#[cfg(test)] +mod postgres_tests; diff --git a/crates/server/src/jobs/account_recovery/postgres_tests.rs b/crates/server/src/jobs/account_recovery/postgres_tests.rs new file mode 100644 index 00000000..6e0a67e1 --- /dev/null +++ b/crates/server/src/jobs/account_recovery/postgres_tests.rs @@ -0,0 +1,1770 @@ +use std::sync::Arc; + +use base64::Engine; +use diesel::sql_types::{BigInt, Jsonb, Text}; +use diesel_async::RunQueryDsl; +use guardian_shared::{ + FromJson, ToJson, + auth_request_payload::AuthRequestPayload, + hex::{FromHex, IntoHex}, +}; +use miden_protocol::account::{ + Account, AccountCodePatch, AccountDelta, AccountStoragePatch, AccountVaultDelta, +}; +use miden_protocol::block::BlockNumber; +use miden_protocol::transaction::{ + InputNotes, RawOutputNotes, TransactionSummary, TransactionSummaryUserParams, +}; +use miden_protocol::{Felt, Word}; + +use crate::delta_object::DeltaStatus; +use crate::metadata::{ + AccountMetadata, MetadataStore, NetworkConfig, postgres::PostgresMetadataStore, +}; +use crate::network::{NetworkType, miden::MidenNetworkClient}; +use crate::services::{GetStateParams, get_state}; +use crate::state_object::StateObject; +use crate::storage::encryption::{ + cipher::{Aes256GcmCipher, StorageCipher}, + decorator::EncryptedStorage, + envelope::RecordAad, + key_provider::InMemoryKeyProvider, +}; +use crate::storage::{StorageBackend, postgres::PostgresService}; +use crate::testing::helpers::{ + IntegrationMockNetworkClient, TestEcdsaSigner, build_guarded_multisig_account, + create_test_app_state_with_mocks, +}; + +use super::*; + +fn account_at_nonce(account: &Account, nonce: Felt) -> Account { + Account::new_existing( + account.id(), + account.vault().clone(), + account.storage().clone(), + account.code().clone(), + nonce, + ) +} + +fn canonical_delta( + network: &dyn NetworkClient, + delta: AccountDelta, + prev_commitment: String, + base_state: &serde_json::Value, +) -> (DeltaObject, AppliedState) { + let account_id = delta.id().to_hex(); + let payload = TransactionSummary::new( + delta, + InputNotes::new(Vec::new()).unwrap(), + RawOutputNotes::new(Vec::new()).unwrap(), + BlockNumber::from(0), + Word::default(), + 0, + TransactionSummaryUserParams::new([Felt::ZERO; 6]), + ) + .to_json(); + let applied = network.apply_delta(base_state, &payload).unwrap(); + let row = DeltaObject { + account_id, + nonce: applied.nonce.unwrap(), + prev_commitment, + new_commitment: Some(applied.commitment.clone()), + delta_payload: payload, + status: DeltaStatus::canonical("2026-11-14T12:00:00Z".into()), + ..Default::default() + }; + (row, applied) +} + +fn canonical_history( + account: &Account, + network: &dyn NetworkClient, +) -> (Vec, AppliedState) { + let (first, first_state) = canonical_delta( + network, + AccountDelta::try_from(account_at_nonce(account, Felt::ONE)).unwrap(), + account.to_commitment().into_hex(), + &serde_json::Value::Null, + ); + let (second, second_state) = canonical_delta( + network, + AccountDelta::new( + account.id(), + AccountStoragePatch::default(), + AccountVaultDelta::default(), + AccountCodePatch::default(), + Felt::ONE, + ) + .unwrap(), + first_state.commitment, + &first_state.state_json, + ); + (vec![second, first], second_state) +} + +fn prove( + network: &dyn NetworkClient, + account_id: &str, + target: &str, + deltas: Vec, +) -> Result<(AppliedState, Auth), String> { + replay(network, account_id, None, deltas) + .and_then(|rebuilt| prove_recovery(network, account_id, target, rebuilt)) +} + +#[tokio::test] +async fn rejects_incomplete_history_and_commitment_mismatches() { + let network = MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal); + let scheme = SignatureScheme::Ecdsa; + let account = build_guarded_multisig_account( + [25; 32], + 1, + &[(Word::from([1, 2, 3, 4_u32]), scheme)], + (Word::from([5, 6, 7, 8_u32]), scheme), + ); + let (mut deltas, expected) = canonical_history(&account, &network); + let account_id = account.id().to_hex(); + let target = &expected.commitment; + assert_eq!( + prove(&network, &account_id, target, deltas.clone()) + .unwrap() + .0, + expected + ); + let first_page = replay(&network, &account_id, None, vec![deltas[1].clone()]).unwrap(); + assert_eq!(first_page.nonce, Some(1)); + assert!(prove_recovery(&network, &account_id, target, first_page.clone()).is_err()); + assert!( + replay( + &network, + &account_id, + Some(first_page.clone()), + vec![deltas[1].clone()] + ) + .is_err() + ); + assert_eq!( + replay( + &network, + &account_id, + Some(first_page), + vec![deltas[0].clone()] + ) + .unwrap(), + expected + ); + assert!(prove(&network, &account_id, target, vec![deltas[0].clone()]).is_err()); + deltas[1].new_commitment = Some("wrong commitment".into()); + assert!(prove(&network, &account_id, target, deltas).is_err()); +} + +#[tokio::test] +async fn rebuilds_the_signer_set_under_each_scheme() { + let network = MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal); + let signer = Word::from([1, 2, 3, 4_u32]); + let commitments = vec![signer.into_hex()]; + let cases = [ + ( + SignatureScheme::Falcon, + SignatureScheme::Falcon, + Auth::MidenFalconRpo { + cosigner_commitments: commitments.clone(), + }, + ), + ( + SignatureScheme::Ecdsa, + SignatureScheme::Ecdsa, + Auth::MidenEcdsa { + cosigner_commitments: commitments.clone(), + }, + ), + ( + SignatureScheme::Falcon, + SignatureScheme::Ecdsa, + Auth::Miden { + cosigners: vec![crate::metadata::auth::MidenCosigner { + commitment: commitments[0].clone(), + scheme: SignatureScheme::Falcon, + }], + guardian_scheme: SignatureScheme::Ecdsa, + }, + ), + ]; + for (signer_scheme, guardian_scheme, expected) in cases { + let account = build_guarded_multisig_account( + [26; 32], + 1, + &[(signer, signer_scheme)], + (Word::from([5, 6, 7, 8_u32]), guardian_scheme), + ); + let (deltas, rebuilt) = canonical_history(&account, &network); + let (_, auth) = prove( + &network, + &account.id().to_hex(), + &rebuilt.commitment, + deltas, + ) + .unwrap(); + assert_eq!(auth, expected); + } +} + +#[test] +fn a_proposal_review_keeps_what_the_restored_signers_proposed_or_approved() { + use crate::delta_object::CosignerSignature; + use crate::metadata::auth::ChainCosigner; + use crate::storage::ProposalRecord; + use crate::testing::helpers::{TestSigner, create_test_delta_payload}; + + let account_id = "0x7b7b7b7a7b7b7b017b7b7b7b7b7b7b"; + let owner = TestSigner::new(); + let other_key = TestSigner::new(); + let tx_summary = create_test_delta_payload(account_id); + let message = TransactionSummary::from_json(&tx_summary) + .unwrap() + .to_commitment(); + let approval = |signer: &TestSigner| CosignerSignature { + signature: guardian_shared::ProposalSignature::Falcon { + signature: signer.sign_word(message), + }, + timestamp: "2026-11-14T12:50:00Z".into(), + signer_id: signer.commitment_hex.clone(), + }; + let payload = |approvals: &[CosignerSignature]| { + let signatures: Vec = approvals + .iter() + .map(|approval| { + serde_json::json!({ + "signer_id": approval.signer_id, + "signature": approval.signature, + }) + }) + .collect(); + serde_json::json!({ "tx_summary": tx_summary, "signatures": signatures }) + }; + let pending = |proposer: &TestSigner, approvals: &[CosignerSignature]| DeltaStatus::Pending { + timestamp: "2026-11-14T12:45:00Z".into(), + proposer_id: proposer.commitment_hex.clone(), + cosigner_sigs: approvals.to_vec(), + }; + let record = + |commitment: &str, proposer: &TestSigner, approvals: &[CosignerSignature]| ProposalRecord { + account_id: account_id.into(), + commitment: commitment.into(), + proposal: DeltaObject { + account_id: account_id.into(), + delta_payload: payload(approvals), + status: pending(proposer, approvals), + ..Default::default() + }, + }; + let restored = |scheme: SignatureScheme| AppliedState { + state_json: serde_json::Value::Null, + commitment: "0xrestored".into(), + nonce: Some(1), + cosigners: vec![ChainCosigner { + commitment: owner.commitment_hex.clone(), + scheme: Some(scheme), + }], + guardian_commitment: None, + }; + let own = approval(&owner); + let other = approval(&other_key); + let mut unreadable = record("unreadable", &other_key, std::slice::from_ref(&other)); + unreadable.proposal.delta_payload = serde_json::json!({ "tx_summary": "not a summary" }); + let records = [ + record("signed", &owner, std::slice::from_ref(&own)), + record("mixed", &owner, &[own.clone(), other.clone()]), + record("approved", &other_key, std::slice::from_ref(&own)), + record("unrelated", &other_key, std::slice::from_ref(&other)), + record("unsigned", &owner, &[]), + unreadable, + ]; + + let changes: Vec = + proposals::review(&restored(SignatureScheme::Falcon), &records) + .into_iter() + .map(|reviewed| reviewed.change) + .collect(); + assert_eq!( + changes, + [ + ProposalChange::Keep, + ProposalChange::DropApprovals { + status: pending(&owner, std::slice::from_ref(&own)), + delta_payload: payload(std::slice::from_ref(&own)), + }, + ProposalChange::Keep, + ProposalChange::Remove, + ProposalChange::Keep, + ProposalChange::Keep, + ] + ); + + let under_another_scheme = proposals::review(&restored(SignatureScheme::Ecdsa), &records[..1]); + assert_eq!( + under_another_scheme[0].change, + ProposalChange::DropApprovals { + status: pending(&owner, &[]), + delta_payload: payload(&[]), + } + ); +} + +#[tokio::test] +async fn partial_history_fixture_cannot_rebuild_a_missing_base() { + let network = MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal); + let delta: DeltaObject = serde_json::from_str(crate::testing::fixtures::DELTA_1_JSON).unwrap(); + let summary = TransactionSummary::from_json(&delta.delta_payload).unwrap(); + assert!(summary.account_delta().code().is_empty()); + assert!( + network + .apply_delta(&serde_json::Value::Null, &delta.delta_payload) + .is_err() + ); +} + +#[tokio::test] +#[ignore = "requires a dedicated Postgres test database"] +async fn account_recovery_restores_verified_state_once_and_preserves_encrypted_evidence() { + let url = crate::testing::pg::test_database_url().await; + let raw = Arc::new(PostgresService::new(&url, 4).await.unwrap()); + let metadata = Arc::new(PostgresMetadataStore::new(&url, 4).await.unwrap()); + let key = base64::engine::general_purpose::STANDARD.encode([2_u8; 32]); + let provider = InMemoryKeyProvider::from_dev_key(&key, "recovery-test").unwrap(); + let cipher = Arc::new(Aes256GcmCipher::new(Arc::new(provider))); + let storage = Arc::new(EncryptedStorage::new(raw, cipher.clone())); + let store = RecoveryStore::new( + metadata.pool_handle(), + Some(cipher.clone()), + "test-recovery", + ); + let contender = RecoveryStore::new( + metadata.pool_handle(), + Some(cipher.clone()), + "other-recovery", + ); + let lease = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + let chain = Arc::new(IntegrationMockNetworkClient::new( + MidenNetworkClient::lazy_for_test(NetworkType::MidenLocal), + )); + let state = create_test_app_state_with_mocks(storage.clone(), chain.clone(), metadata.clone()); + let scheme = SignatureScheme::Ecdsa; + let signer = TestEcdsaSigner::new(); + let account = build_guarded_multisig_account( + [25; 32], + 1, + &[(Word::from_hex(&signer.commitment_hex).unwrap(), scheme)], + ( + Word::from_hex(&state.ack.commitment(&scheme)).unwrap(), + scheme, + ), + ); + let (deltas, canonical) = canonical_history(&account, chain.as_ref()); + let account_id = &deltas[0].account_id; + let target = canonical.commitment.as_str(); + let old_auth = Auth::MidenEcdsa { + cosigner_commitments: vec![Word::from([8, 7, 6, 5_u32]).into_hex()], + }; + let expected_auth = Auth::MidenEcdsa { + cosigner_commitments: vec![signer.commitment_hex.clone()], + }; + let other_state = account_at_nonce(&account, Felt::from(2_u8)); + let prior = StateObject { + account_id: account_id.clone(), + state_json: other_state.to_json(), + commitment: other_state.to_commitment().into_hex(), + nonce: Some(2), + created_at: "2026-11-14T12:00:00Z".into(), + updated_at: "2026-11-14T13:00:00Z".into(), + auth_scheme: "ecdsa".into(), + }; + metadata + .set(AccountMetadata { + account_id: account_id.clone(), + auth: old_auth.clone(), + network_config: NetworkConfig::miden_default(), + created_at: prior.created_at.clone(), + updated_at: prior.updated_at.clone(), + has_pending_candidate: false, + paused_at: None, + paused_reason: None, + released_at: None, + }) + .await + .unwrap(); + storage + .submit_state(&StateObject { + state_json: canonical.state_json.clone(), + commitment: canonical.commitment.clone(), + updated_at: prior.created_at.clone(), + ..prior.clone() + }) + .await + .unwrap(); + storage.replace_state(&prior, target).await.unwrap(); + for delta in &deltas { + storage.submit_delta(delta).await.unwrap(); + } + let testnet = NetworkConfig::Miden { + network_type: MidenNetworkType::Testnet, + }; + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("INSERT INTO account_recovery_startup (network_config) VALUES ($1)") + .bind::(serde_json::to_value(&testnet).unwrap()) + .execute(&mut conn) + .await + .unwrap(); + for label in [&testnet, &NetworkConfig::miden_default()] { + diesel::sql_query( + "UPDATE account_metadata SET network_config = $2 WHERE account_id = $1", + ) + .bind::(account_id) + .bind::(serde_json::to_value(label).unwrap()) + .execute(&mut conn) + .await + .unwrap(); + let candidates = store + .candidates(MidenNetworkType::Mainnet, "", 10) + .await + .unwrap(); + assert_eq!(candidates.len(), usize::from(label != &testnet)); + } + } + assert_eq!( + storage.pull_state_head(account_id).await.unwrap().nonce, + None + ); + assert_eq!( + metadata + .get_for_read(account_id) + .await + .unwrap() + .unwrap() + .auth, + old_auth + ); + let mut candidates = store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap(); + assert_eq!(candidates.len(), 1); + let candidate = candidates.pop().unwrap(); + assert_eq!(candidate.canonical_nonce, Some(2)); + assert_eq!(candidate.stored_nonce, None); + + let mut absent = state.clone(); + absent.network_client = Arc::new( + crate::testing::mocks::MockNetworkClient::new() + .with_verify_commitment(Ok(StateVerification::Absent)) + .with_verify_commitment(Ok(StateVerification::Absent)), + ); + assert!(matches!( + check_candidate(&absent, &store, &candidate).await.unwrap(), + RecoveryOutcome::Retryable(reason) if reason.contains("no record of the account") + )); + let mut undeployed = store + .candidates(MidenNetworkType::Local, "", 10) + .await + .unwrap() + .pop() + .unwrap(); + undeployed.canonical_nonce = None; + undeployed.canonical_commitment = None; + let absent_with = |first_state: Result<(), String>| { + let mut absent = state.clone(); + absent.network_client = Arc::new( + crate::testing::mocks::MockNetworkClient::new() + .with_verify_commitment(Ok(StateVerification::Absent)) + .with_validate_first_state(first_state), + ); + absent + }; + for stored_nonce in [Some(0), None] { + undeployed.stored_nonce = stored_nonce; + assert!(matches!( + check_candidate(&absent_with(Ok(())), &store, &undeployed) + .await + .unwrap(), + RecoveryOutcome::Unchanged + )); + assert!(matches!( + check_candidate(&absent_with(Err("nonce is 5, not 0".into())), &store, &undeployed) + .await + .unwrap(), + RecoveryOutcome::Unresolved(reason) if reason.contains("no record of the account") + )); + } + undeployed.stored_nonce = Some(5); + assert!(matches!( + check_candidate(&absent_with(Ok(())), &store, &undeployed) + .await + .unwrap(), + RecoveryOutcome::Unresolved(reason) if reason.contains("no record of the account") + )); + + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("BEGIN").execute(&mut conn).await.unwrap(); + diesel::sql_query("LOCK TABLE states IN ACCESS EXCLUSIVE MODE") + .execute(&mut conn) + .await + .unwrap(); + let timed_out = run( + &state, + &store, + MidenNetworkType::Testnet, + ScanWrites::Apply(&lease), + &CancellationToken::new(), + "", + Instant::now() + Duration::from_millis(40), + ) + .await + .unwrap(); + diesel::sql_query("ROLLBACK") + .execute(&mut conn) + .await + .unwrap(); + assert!(timed_out.stopped_early); + assert_eq!(timed_out.checked, 0); + } + + for page in [1, 2, HISTORY_PAGE] { + assert!(matches!( + rebuild(&state, account_id, 2, page).await.unwrap(), + Rebuild::Rebuilt(rebuilt) if rebuilt.commitment == target && rebuilt.nonce == Some(2) + )); + } + assert!(matches!( + rebuild(&state, account_id, 3, 1).await.unwrap(), + Rebuild::Unresolved(reason) if reason.contains("not contiguous") + )); + + chain.set_on_chain_commitment(account_id, "different head"); + assert!(matches!( + check_candidate(&state, &store, &candidate).await.unwrap(), + RecoveryOutcome::Unresolved(reason) if reason.contains("missing from the canonical history") + )); + assert_ne!(deltas[1].prev_commitment, prior.commitment); + for earlier in [ + deltas[1].new_commitment.as_deref().unwrap(), + deltas[1].prev_commitment.as_str(), + ] { + chain.set_on_chain_commitment(account_id, earlier); + assert!(matches!( + check_candidate(&state, &store, &candidate).await.unwrap(), + RecoveryOutcome::Retryable(reason) if reason.contains("earlier state") + )); + } + chain.set_on_chain_commitment(account_id, target); + let pending = DeltaObject { + nonce: 3, + prev_commitment: target.into(), + new_commitment: None, + status: DeltaStatus::candidate("2026-11-14T12:30:00Z".into()), + ..deltas[0].clone() + }; + storage.submit_delta(&pending).await.unwrap(); + let current_candidate = || async { + store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap() + .pop() + .unwrap() + }; + assert_eq!( + current_candidate().await.unsettled_work.as_deref(), + Some("a delta awaits its on-chain outcome") + ); + assert!(matches!( + check_candidate(&state, &store, &candidate).await.unwrap(), + RecoveryOutcome::Verified(_) + )); + assert!( + !store + .apply( + &candidate, + &canonical.state_json, + &expected_auth, + &ProposalReview::Reviewed(Vec::new()), + state.clock.now(), + &lease + ) + .await + .unwrap() + ); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment + ); + assert!(matches!( + run_within( + &state, + &contender, + MidenNetworkType::Testnet, + SCAN_BUDGET, + Duration::ZERO + ) + .await, + Err(StartupRecoveryError::LeaseBusy) + )); + store.leader.release(lease.clone()).await.unwrap(); + + run_once(&state, &store, MidenNetworkType::Testnet) + .await + .unwrap(); + assert!( + !store + .startup_completed(MidenNetworkType::Testnet) + .await + .unwrap() + ); + + storage + .update_delta_status( + account_id, + 3, + DeltaStatus::discarded_client_abandoned("2026-11-14T12:40:00Z".into()), + ) + .await + .unwrap(); + storage + .submit_delta(&DeltaObject { + nonce: 4, + prev_commitment: prior.commitment.clone(), + new_commitment: None, + status: DeltaStatus::candidate("2026-11-14T12:41:00Z".into()), + ..deltas[0].clone() + }) + .await + .unwrap(); + assert_eq!(current_candidate().await.unsettled_work, None); + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "INSERT INTO execution_reservations + (account_id, proposal_id, attempt, holder_id, lease_name, fence_token, + lease_expires_at, phase) + VALUES ($1, '0xreserved', 1, 'holder', 'execution', 1, now(), 'accepted')", + ) + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + diesel::sql_query( + "INSERT INTO execution_submissions + (account_id, proposal_id, attempt, candidate_nonce, transaction_id, + expected_commitment, reference_block, expiration_block, base_commitment) + VALUES ($1, '0xreserved', 1, 3, '0xtx', '0xexpected', 1, 2, $2)", + ) + .bind::(account_id) + .bind::(target) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!( + current_candidate().await.unsettled_work.as_deref(), + Some("an execution is reserved") + ); + diesel::sql_query( + "UPDATE execution_submissions SET base_commitment = $2 WHERE account_id = $1", + ) + .bind::(account_id) + .bind::(&prior.commitment) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(current_candidate().await.unsettled_work, None); + + diesel::sql_query( + "UPDATE deltas SET new_commitment = '0xlanded' WHERE account_id = $1 AND nonce = 4", + ) + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + chain.set_on_chain_commitment(account_id, "0xlanded"); + assert!(matches!( + check_candidate(&state, &store, ¤t_candidate().await) + .await + .unwrap(), + RecoveryOutcome::Retryable(reason) if reason.contains("landed") + )); + chain.set_on_chain_commitment(account_id, target); + } + + assert!( + store + .complete_startup(MidenNetworkType::Testnet, 0, &lease) + .await + .is_err() + ); + + let mut unavailable = state.clone(); + unavailable.network_client = Arc::new( + crate::testing::mocks::MockNetworkClient::new() + .with_verify_commitment(Err("simulated RPC outage".into())), + ); + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("DELETE FROM account_recovery_pending WHERE account_id = $1") + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + run_once(&unavailable, &store, MidenNetworkType::Testnet) + .await + .unwrap(); + assert!( + !store + .startup_completed(MidenNetworkType::Testnet) + .await + .unwrap() + ); + + #[derive(diesel::QueryableByName)] + struct DeferredReason { + #[diesel(sql_type = Text)] + reason: String, + } + let deferred = diesel::sql_query( + "SELECT reason FROM account_recovery_pending WHERE account_id = $1 AND outcome = 'deferred'", + ) + .bind::(account_id) + .get_result::(&mut conn) + .await + .unwrap(); + assert!(deferred.reason.contains("account verification failed")); + + diesel::sql_query("BEGIN").execute(&mut conn).await.unwrap(); + diesel::sql_query("LOCK TABLE account_recovery_backups IN ACCESS EXCLUSIVE MODE") + .execute(&mut conn) + .await + .unwrap(); + let blocked_restore = timeout( + Duration::from_secs(8), + run_once(&state, &store, MidenNetworkType::Testnet), + ) + .await; + diesel::sql_query("ROLLBACK") + .execute(&mut conn) + .await + .unwrap(); + blocked_restore.unwrap().unwrap(); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment + ); + let deferred = diesel::sql_query( + "SELECT reason FROM account_recovery_pending WHERE account_id = $1 AND outcome = 'deferred'", + ) + .bind::(account_id) + .get_result::(&mut conn) + .await + .unwrap(); + assert!(deferred.reason.contains("restore transaction failed")); + drop(conn); + + let other_key = crate::testing::helpers::TestSigner::new(); + let tx_summary = crate::testing::helpers::create_test_delta_payload(account_id); + let message = TransactionSummary::from_json(&tx_summary) + .unwrap() + .to_commitment(); + let approval = |signature: guardian_shared::ProposalSignature, signer_id: &str| { + crate::delta_object::CosignerSignature { + signature, + timestamp: "2026-11-14T12:50:00Z".into(), + signer_id: signer_id.into(), + } + }; + let other_approval = approval( + guardian_shared::ProposalSignature::Falcon { + signature: other_key.sign_word(message), + }, + &other_key.commitment_hex, + ); + let signer_approval = approval( + guardian_shared::ProposalSignature::Ecdsa { + signature: signer.sign_word(message), + public_key: Some(signer.pubkey_hex.clone()), + message_format: guardian_shared::EcdsaMessageFormat::Raw, + }, + &signer.commitment_hex, + ); + let pending_proposal = + |nonce: u64, proposer: &str, approvals: &[crate::delta_object::CosignerSignature]| { + let signatures: Vec = approvals + .iter() + .map(|approval| { + serde_json::json!({ + "signer_id": approval.signer_id, + "signature": approval.signature, + }) + }) + .collect(); + DeltaObject { + account_id: account_id.clone(), + nonce, + prev_commitment: prior.commitment.clone(), + delta_payload: serde_json::json!({ + "tx_summary": tx_summary, + "signatures": signatures, + }), + status: DeltaStatus::Pending { + timestamp: "2026-11-14T12:45:00Z".into(), + proposer_id: proposer.into(), + cosigner_sigs: approvals.to_vec(), + }, + ..Default::default() + } + }; + let untouched = pending_proposal( + 6, + &signer.commitment_hex, + std::slice::from_ref(&signer_approval), + ); + for (commitment, proposal) in [ + ( + "0xproposed-by-another-key", + pending_proposal( + 4, + &other_key.commitment_hex, + std::slice::from_ref(&other_approval), + ), + ), + ( + "0xproposed-by-the-signer", + pending_proposal( + 5, + &signer.commitment_hex, + &[signer_approval.clone(), other_approval.clone()], + ), + ), + ("0xsigned-by-the-signer-only", untouched.clone()), + ] { + storage + .submit_delta_proposal(commitment, &proposal) + .await + .unwrap(); + } + + #[derive(diesel::QueryableByName, Debug, PartialEq, Eq)] + struct RecoveryRows { + #[diesel(sql_type = BigInt)] + pending: i64, + #[diesel(sql_type = BigInt)] + backups: i64, + #[diesel(sql_type = Text)] + progress: String, + } + let recovery_rows = || async { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "SELECT (SELECT count(*) FROM account_recovery_pending) AS pending, + (SELECT count(*) FROM account_recovery_backups) AS backups, + (SELECT COALESCE(string_agg(to_jsonb(s)::text, ',' ORDER BY s.network_config::text), '') + FROM account_recovery_startup s) AS progress", + ) + .get_result::(&mut conn) + .await + .unwrap() + }; + let before_dry_run = recovery_rows().await; + assert_eq!((before_dry_run.pending, before_dry_run.backups), (1, 0)); + dry_run(&state, &store, MidenNetworkType::Testnet) + .await + .unwrap(); + assert_eq!(recovery_rows().await, before_dry_run); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment + ); + assert_eq!( + storage + .pull_pending_proposals(account_id) + .await + .unwrap() + .len(), + 3 + ); + + let reviewing = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + let reviewed_candidate = store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap() + .pop() + .unwrap(); + let RecoveryOutcome::Verified(reviewed) = check_candidate(&state, &store, &reviewed_candidate) + .await + .unwrap() + else { + panic!("the account is restorable"); + }; + assert_eq!(reviewed.changed_proposals().count(), 2); + storage + .submit_delta_proposal( + "0xadded-after-the-review", + &pending_proposal(7, &signer.commitment_hex, &[]), + ) + .await + .unwrap(); + assert!( + !store + .apply( + &reviewed_candidate, + &reviewed.state.state_json, + &reviewed.auth, + &reviewed.proposals, + state.clock.now(), + &reviewing, + ) + .await + .unwrap() + ); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment + ); + storage + .delete_delta_proposal(account_id, "0xadded-after-the-review") + .await + .unwrap(); + store.leader.release(reviewing).await.unwrap(); + metadata + .set_pause(account_id, state.clock.now(), "operator review") + .await + .unwrap(); + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("UPDATE states SET nonce = 9 WHERE account_id = $1") + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + } + assert_eq!( + storage.pull_state_head(account_id).await.unwrap().nonce, + Some(9) + ); + let eligible = store + .candidates(MidenNetworkType::Testnet, "", 10) + .await + .unwrap() + .pop() + .unwrap(); + assert!( + !store + .apply( + &eligible, + &reviewed.state.state_json, + &reviewed.auth, + &reviewed.proposals, + state.clock.now(), + &lease, + ) + .await + .unwrap() + ); + assert_eq!( + storage.pull_state(account_id).await.unwrap().commitment, + prior.commitment + ); + { + #[derive(diesel::QueryableByName)] + struct Base { + #[diesel(sql_type = Text)] + prev_commitment: String, + } + let reserved_proposal = "0xproposed-by-the-signer"; + let rebase = |prev_commitment: String| { + let metadata = metadata.clone(); + async move { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "UPDATE delta_proposals SET prev_commitment = $3 + WHERE account_id = $1 AND commitment = $2", + ) + .bind::(account_id) + .bind::(reserved_proposal) + .bind::(prev_commitment) + .execute(&mut conn) + .await + .unwrap() + } + }; + let original = { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("DELETE FROM execution_submissions WHERE account_id = $1") + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + diesel::sql_query( + "UPDATE execution_reservations SET proposal_id = $2 WHERE account_id = $1", + ) + .bind::(account_id) + .bind::(reserved_proposal) + .execute(&mut conn) + .await + .unwrap(); + diesel::sql_query( + "SELECT prev_commitment FROM delta_proposals + WHERE account_id = $1 AND commitment = $2", + ) + .bind::(account_id) + .bind::(reserved_proposal) + .get_result::(&mut conn) + .await + .unwrap() + .prev_commitment + }; + assert_ne!(original, target); + assert_eq!(current_candidate().await.unsettled_work, None); + + assert_eq!(rebase(target.to_owned()).await, 1); + assert_eq!( + current_candidate().await.unsettled_work.as_deref(), + Some("an execution is reserved") + ); + let current = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + assert!( + !store + .apply( + &eligible, + &reviewed.state.state_json, + &reviewed.auth, + &reviewed.proposals, + state.clock.now(), + ¤t, + ) + .await + .unwrap() + ); + store.leader.release(current).await.unwrap(); + assert_eq!(rebase(original).await, 1); + assert_eq!(current_candidate().await.unsettled_work, None); + } + storage + .submit_delta_proposal( + "0xunreadable-status", + &pending_proposal(8, &other_key.commitment_hex, &[]), + ) + .await + .unwrap(); + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + let updated = diesel::sql_query( + "UPDATE delta_proposals SET status = '{\"status\": \"pending\", \"cosigner_sigs\": 1}' + WHERE account_id = $1 AND commitment = '0xunreadable-status'", + ) + .bind::(account_id) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(updated, 1); + } + + run_once(&state, &store, MidenNetworkType::Testnet) + .await + .unwrap(); + assert!( + store + .startup_completed(MidenNetworkType::Testnet) + .await + .unwrap() + ); + assert!( + !store + .startup_completed(MidenNetworkType::Local) + .await + .unwrap() + ); + assert_eq!( + storage.pull_state_head(account_id).await.unwrap().nonce, + Some(2) + ); + + let kept = storage.pull_pending_proposals(account_id).await.unwrap(); + assert_eq!( + kept.iter() + .map(|record| record.commitment.as_str()) + .collect::>(), + ["0xproposed-by-the-signer", "0xsigned-by-the-signer-only"] + ); + { + #[derive(diesel::QueryableByName)] + struct Rows { + #[diesel(sql_type = BigInt)] + total: i64, + } + let mut conn = metadata.pool_handle().get().await.unwrap(); + let unreadable = diesel::sql_query( + "SELECT count(*) AS total FROM delta_proposals + WHERE account_id = $1 AND commitment = '0xunreadable-status'", + ) + .bind::(account_id) + .get_result::(&mut conn) + .await + .unwrap(); + assert_eq!(unreadable.total, 1); + } + let trimmed = pending_proposal( + 5, + &signer.commitment_hex, + std::slice::from_ref(&signer_approval), + ); + assert_eq!( + (&kept[0].proposal.status, &kept[0].proposal.delta_payload), + (&trimmed.status, &trimmed.delta_payload) + ); + assert_eq!( + (&kept[1].proposal.status, &kept[1].proposal.delta_payload), + (&untouched.status, &untouched.delta_payload) + ); + + let held = contender + .leader + .try_acquire(LEASE_TTL) + .await + .unwrap() + .unwrap(); + assert!(matches!( + run_within( + &state, + &store, + MidenNetworkType::Testnet, + SCAN_BUDGET, + Duration::ZERO + ) + .await, + Err(StartupRecoveryError::LeaseBusy) + )); + assert!(contender.leader.verify_held(&held).await.unwrap()); + contender.leader.release(held).await.unwrap(); + run_once(&state, &store, MidenNetworkType::Testnet) + .await + .unwrap(); + assert!( + !store + .apply( + &candidate, + &canonical.state_json, + &expected_auth, + &ProposalReview::Reviewed(Vec::new()), + state.clock.now(), + &lease + ) + .await + .unwrap() + ); + let payload = + AuthRequestPayload::from_json_serializable(&serde_json::json!({"account_id": account_id})) + .unwrap(); + let (signature, timestamp) = signer.sign_request(account_id, &payload); + let restored = get_state( + &state, + GetStateParams { + account_id: account_id.clone(), + credentials: crate::metadata::Credentials::signature( + signer.pubkey_hex, + signature, + timestamp, + ) + .with_request_payload(payload), + }, + ) + .await + .unwrap() + .state; + assert_eq!( + chain + .get_state_head(account_id, &restored.state_json) + .unwrap() + .commitment, + target + ); + let restored_metadata = metadata.get_for_read(account_id).await.unwrap().unwrap(); + assert_eq!(restored_metadata.auth, expected_auth); + assert!(restored_metadata.paused_at.is_some()); + assert_eq!( + storage.pull_state_head(account_id).await.unwrap().nonce, + Some(2) + ); + assert_eq!( + restored_metadata.paused_reason.as_deref(), + Some("operator review") + ); + + #[derive(diesel::QueryableByName)] + struct Backup { + #[diesel(sql_type = BigInt)] + total: i64, + #[diesel(sql_type = Jsonb)] + prior_state: serde_json::Value, + #[diesel(sql_type = Jsonb)] + prior_auth: serde_json::Value, + #[diesel(sql_type = Jsonb)] + prior_proposals: serde_json::Value, + } + let mut conn = metadata.pool_handle().get().await.unwrap(); + let backup = diesel::sql_query("SELECT count(*) OVER () AS total, prior_state, prior_auth, prior_proposals FROM account_recovery_backups WHERE account_id = $1") + .bind::(account_id).get_result::(&mut conn).await.unwrap(); + let old_payload = &backup.prior_state["state_json"]; + assert_ne!(old_payload, &prior.state_json); + assert_eq!( + cipher + .decrypt(&RecordAad::State { account_id }, old_payload) + .unwrap(), + prior.state_json + ); + assert_eq!( + serde_json::from_value::(backup.prior_auth).unwrap(), + old_auth + ); + assert_eq!(backup.total, 1); + let saved_proposals: Vec<&str> = backup + .prior_proposals + .as_array() + .unwrap() + .iter() + .map(|proposal| proposal["commitment"].as_str().unwrap()) + .collect(); + assert_eq!( + saved_proposals, + ["0xproposed-by-another-key", "0xproposed-by-the-signer"] + ); + + #[derive(diesel::QueryableByName)] + struct PendingCount { + #[diesel(sql_type = BigInt)] + pending: i64, + } + let pending = diesel::sql_query( + "SELECT count(*) AS pending FROM account_recovery_pending WHERE account_id = $1", + ) + .bind::(account_id) + .get_result::(&mut conn) + .await + .unwrap(); + assert_eq!(pending.pending, 0); +} + +fn scan_progress(resume_after: &str) -> ScanProgress { + ScanProgress { + resume_after: resume_after.into(), + ..Default::default() + } +} + +#[test] +fn a_scan_can_complete_across_startups_when_the_pass_deferred_no_account() { + let interrupted = ScanSummary { + unresolved: 2, + last_checked: Some("0xb".into()), + stopped_early: true, + ..Default::default() + }; + assert_eq!( + interrupted.next_step(scan_progress("0xa")), + ScanStep::Resume(ScanProgress { + unresolved: 2, + ..scan_progress("0xb") + }) + ); + + let stalled = ScanSummary { + stopped_early: true, + ..Default::default() + }; + assert_eq!( + stalled.next_step(scan_progress("0xa")), + ScanStep::Resume(scan_progress("0xa")) + ); + + let finished = ScanSummary { + unresolved: 1, + last_checked: Some("0xc".into()), + ..Default::default() + }; + assert_eq!( + finished.next_step(ScanProgress::default()), + ScanStep::Completed { unresolved: 1 } + ); + assert_eq!( + finished.next_step(ScanProgress { + unresolved: 2, + ..scan_progress("0xa") + }), + ScanStep::Completed { unresolved: 3 } + ); + assert_eq!( + finished.next_step(ScanProgress { + deferred: true, + ..scan_progress("0xa") + }), + ScanStep::Resume(ScanProgress::default()) + ); + + let rolled_back = ScanSummary { + failed: 3, + failed_ahead_of_position: 3, + last_checked: Some("0xa".into()), + stopped_early: true, + ..Default::default() + }; + assert_eq!( + rolled_back.next_step(ScanProgress::default()), + ScanStep::Resume(scan_progress("0xa")) + ); + let failed_behind_position = ScanSummary { + failed: 4, + ..rolled_back + }; + assert_eq!( + failed_behind_position.next_step(ScanProgress::default()), + ScanStep::Resume(ScanProgress { + deferred: true, + ..scan_progress("0xa") + }) + ); + + for deferring in [ + ScanSummary { + retryable: 1, + ..Default::default() + }, + ScanSummary { + failed: 1, + ..Default::default() + }, + ] { + assert_eq!( + deferring.next_step(ScanProgress::default()), + ScanStep::Resume(ScanProgress::default()) + ); + } +} + +#[tokio::test] +#[ignore = "requires a dedicated Postgres test database"] +async fn account_recovery_scan_continues_from_its_saved_position_and_records_pending_accounts() { + let url = crate::testing::pg::test_database_url().await; + let storage = Arc::new(PostgresService::new(&url, 4).await.unwrap()); + let metadata = Arc::new(PostgresMetadataStore::new(&url, 4).await.unwrap()); + let store = RecoveryStore::new(metadata.pool_handle(), None, "resume-recovery"); + let network = MidenNetworkType::Cedar; + let first = "0xresume0000000000000000000000a"; + let second = "0xresume0000000000000000000000b"; + let joined_later = "0xresume0000000000000000000000c"; + for account_id in [first, second, joined_later] { + metadata + .set(AccountMetadata { + account_id: account_id.into(), + auth: Auth::MidenEcdsa { + cosigner_commitments: vec![Word::from([1, 2, 3, 4_u32]).into_hex()], + }, + network_config: NetworkConfig::Miden { + network_type: network, + }, + created_at: "2026-11-14T12:00:00Z".into(), + updated_at: "2026-11-14T12:00:00Z".into(), + has_pending_candidate: false, + paused_at: None, + paused_reason: None, + released_at: None, + }) + .await + .unwrap(); + storage + .submit_state(&StateObject { + account_id: account_id.into(), + state_json: serde_json::json!({}), + commitment: format!("{account_id}-stored"), + nonce: Some(1), + created_at: "2026-11-14T12:00:00Z".into(), + updated_at: "2026-11-14T13:00:00Z".into(), + auth_scheme: "ecdsa".into(), + }) + .await + .unwrap(); + } + for (nonce, prev_commitment, new_commitment) in [ + (3, "0xheld-base", "0xheld-middle"), + (4, "0xheld-middle", "0xheld-head"), + ] { + storage + .submit_delta(&DeltaObject { + account_id: joined_later.into(), + nonce, + prev_commitment: prev_commitment.into(), + new_commitment: Some(new_commitment.into()), + delta_payload: serde_json::json!({}), + status: DeltaStatus::canonical("2026-11-14T12:30:00Z".into()), + ..Default::default() + }) + .await + .unwrap(); + } + let chain_head = |account_id: &str| StateVerification::Mismatch { + on_chain: if account_id == joined_later { + "0xheld-head".into() + } else { + "0xelsewhere".into() + }, + }; + let checking = |accounts: &[&str]| { + let chain = accounts.iter().rev().fold( + crate::testing::mocks::MockNetworkClient::new(), + |chain, account_id| chain.with_verify_commitment(Ok(chain_head(account_id))), + ); + create_test_app_state_with_mocks(storage.clone(), Arc::new(chain), metadata.clone()) + }; + let lease = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + let later = || Instant::now() + Duration::from_secs(60); + + let whole = run( + &checking(&[first, second, joined_later]), + &store, + network, + ScanWrites::Apply(&lease), + &CancellationToken::new(), + "", + later(), + ) + .await + .unwrap(); + assert_eq!((whole.checked, whole.unresolved), (3, 3)); + assert_eq!(whole.last_checked.as_deref(), Some(joined_later)); + assert!(!whole.stopped_early); + + #[derive(diesel::QueryableByName, Debug, PartialEq, Eq)] + struct Pending { + #[diesel(sql_type = Text)] + account_id: String, + #[diesel(sql_type = Text)] + outcome: String, + #[diesel(sql_type = Text)] + reason: String, + #[diesel(sql_type = diesel::sql_types::Nullable)] + canonical_nonce: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + history_starts_at_nonce: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + history_base_commitment: Option, + } + let pending_accounts = || async { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "SELECT account_id, outcome, reason, canonical_nonce, history_starts_at_nonce, + history_base_commitment + FROM account_recovery_pending WHERE account_id LIKE '0xresume%' ORDER BY account_id", + ) + .load::(&mut conn) + .await + .unwrap() + }; + let without_history = |account_id: &str| Pending { + account_id: account_id.into(), + outcome: "unresolved".into(), + reason: "no canonical history supplies a recovery target".into(), + canonical_nonce: None, + history_starts_at_nonce: None, + history_base_commitment: None, + }; + assert_eq!( + pending_accounts().await, + [ + without_history(first), + without_history(second), + Pending { + account_id: joined_later.into(), + outcome: "unresolved".into(), + reason: "missing canonical deltas from account creation".into(), + canonical_nonce: Some(4), + history_starts_at_nonce: Some(3), + history_base_commitment: Some("0xheld-base".into()), + }, + ] + ); + { + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query("BEGIN").execute(&mut conn).await.unwrap(); + diesel::sql_query("LOCK TABLE account_recovery_pending, account_recovery_startup IN ACCESS EXCLUSIVE MODE") + .execute(&mut conn) + .await + .unwrap(); + let clear = timeout(Duration::from_secs(5), store.clear_pending(second, &lease)).await; + let progress = ScanProgress::default(); + let save = timeout( + Duration::from_secs(5), + store.save_progress(network, &progress, &lease), + ) + .await; + diesel::sql_query("ROLLBACK") + .execute(&mut conn) + .await + .unwrap(); + assert!(clear.unwrap().is_err()); + assert!(save.unwrap().is_err()); + } + store.clear_pending(second, &lease).await.unwrap(); + assert_eq!(pending_accounts().await.len(), 2); + + let tail = run( + &checking(&[second, joined_later]), + &store, + network, + ScanWrites::Apply(&lease), + &CancellationToken::new(), + first, + later(), + ) + .await + .unwrap(); + assert_eq!((tail.checked, tail.unresolved), (2, 2)); + assert_eq!(tail.last_checked.as_deref(), Some(joined_later)); + assert_eq!(pending_accounts().await.len(), 3); + assert_eq!(store.remaining_candidates(network, first).await.unwrap(), 2); + + let unreachable = { + let chain = (0..3).fold( + crate::testing::mocks::MockNetworkClient::new(), + |chain, _| chain.with_verify_commitment(Err("simulated RPC outage".into())), + ); + create_test_app_state_with_mocks(storage.clone(), Arc::new(chain), metadata.clone()) + }; + let outage = run( + &unreachable, + &store, + network, + ScanWrites::Apply(&lease), + &CancellationToken::new(), + "", + later(), + ) + .await + .unwrap(); + assert_eq!((outage.checked, outage.failed), (3, 3)); + assert!(outage.stopped_early); + assert_eq!(outage.last_checked, None); + assert_eq!( + outage.next_step(ScanProgress::default()), + ScanStep::Resume(ScanProgress::default()) + ); + + let flaky = { + let chain = crate::testing::mocks::MockNetworkClient::new() + .with_verify_commitment(Err("simulated RPC error".into())) + .with_verify_commitment(Ok(chain_head(second))) + .with_verify_commitment(Err("simulated RPC error".into())); + create_test_app_state_with_mocks(storage.clone(), Arc::new(chain), metadata.clone()) + }; + let interleaved = run( + &flaky, + &store, + network, + ScanWrites::Apply(&lease), + &CancellationToken::new(), + "", + later(), + ) + .await + .unwrap(); + assert_eq!((interleaved.checked, interleaved.failed), (3, 2)); + assert!(!interleaved.stopped_early); + assert_eq!(interleaved.last_checked.as_deref(), Some(joined_later)); + + assert_eq!( + store.progress(network).await.unwrap(), + ScanProgress::default() + ); + let saved = ScanProgress { + unresolved: 1, + ..scan_progress(first) + }; + store.save_progress(network, &saved, &lease).await.unwrap(); + assert_eq!(store.progress(network).await.unwrap(), saved); + store.leader.release(lease.clone()).await.unwrap(); + assert!(store.save_progress(network, &saved, &lease).await.is_err()); + + let stopped = RecoveryStore::new(metadata.pool_handle(), None, "stopped-recovery"); + stopped + .leader + .try_acquire(Duration::from_secs(60)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + run_within( + &checking(&[]), + &store, + network, + Duration::ZERO, + Duration::ZERO + ) + .await, + Err(StartupRecoveryError::LeaseBusy) + )); + let waited_from = Instant::now(); + assert!(matches!( + run_within( + &checking(&[]), + &store, + network, + Duration::ZERO, + Duration::from_secs(3) + ) + .await, + Err(StartupRecoveryError::LeaseBusy) + )); + assert!(waited_from.elapsed() >= LEASE_RETRY_INTERVAL); + let mut conn = metadata.pool_handle().get().await.unwrap(); + diesel::sql_query( + "UPDATE worker_leases SET expires_at = clock_timestamp() + interval '1 second' + WHERE lease_name = 'account_recovery'", + ) + .execute(&mut conn) + .await + .unwrap(); + drop(conn); + run_within( + &checking(&[]), + &store, + network, + Duration::ZERO, + Duration::from_secs(10), + ) + .await + .unwrap(); + assert_eq!(store.progress(network).await.unwrap(), saved); + + run_within( + &checking(&[second, joined_later]), + &store, + network, + Duration::from_secs(60), + Duration::ZERO, + ) + .await + .unwrap(); + assert_eq!( + store.progress(network).await.unwrap(), + ScanProgress::default() + ); + assert!(store.startup_completed(network).await.unwrap()); + + #[derive(diesel::QueryableByName)] + struct Completion { + #[diesel(sql_type = BigInt)] + unresolved_accounts: i64, + } + let mut conn = metadata.pool_handle().get().await.unwrap(); + let completion = diesel::sql_query( + "SELECT unresolved_accounts FROM account_recovery_startup + WHERE network_config = $1 AND completed_at IS NOT NULL", + ) + .bind::( + serde_json::to_value(NetworkConfig::Miden { + network_type: network, + }) + .unwrap(), + ) + .get_result::(&mut conn) + .await + .unwrap(); + assert_eq!(completion.unresolved_accounts, 3); + + diesel::sql_query("UPDATE states SET nonce = 4 WHERE account_id = $1") + .bind::(joined_later) + .execute(&mut conn) + .await + .unwrap(); + let same_nonce = store + .candidates(network, second, 10) + .await + .unwrap() + .pop() + .unwrap(); + assert_eq!( + (same_nonce.stored_nonce, same_nonce.canonical_nonce), + (Some(4), Some(4)) + ); + let lease = store + .leader + .try_acquire(Duration::from_secs(300)) + .await + .unwrap() + .unwrap(); + store + .record_pending( + &same_nonce, + PendingOutcome::Deferred, + "recorded before the restore", + &lease, + ) + .await + .unwrap(); + assert!( + store + .apply( + &same_nonce, + &serde_json::json!({"restored": true}), + &Auth::MidenEcdsa { + cosigner_commitments: vec![Word::from([1, 2, 3, 4_u32]).into_hex()], + }, + &ProposalReview::Reviewed(vec![ReviewedProposal { + commitment: "0xunreadable-status".into(), + status: DeltaStatus::default(), + change: ProposalChange::Keep, + }]), + chrono::Utc::now(), + &lease, + ) + .await + .unwrap() + ); + let restored = storage.pull_state_head(joined_later).await.unwrap(); + assert_eq!( + (restored.commitment.as_str(), restored.nonce), + ("0xheld-head", Some(4)) + ); + #[derive(diesel::QueryableByName)] + struct PendingRows { + #[diesel(sql_type = BigInt)] + total: i64, + } + let mut conn = metadata.pool_handle().get().await.unwrap(); + let pending = diesel::sql_query( + "SELECT count(*) AS total FROM account_recovery_pending WHERE account_id = $1", + ) + .bind::(joined_later) + .get_result::(&mut conn) + .await + .unwrap(); + assert_eq!(pending.total, 0); + store.leader.release(lease).await.unwrap(); +} diff --git a/crates/server/src/jobs/account_recovery/proposals.rs b/crates/server/src/jobs/account_recovery/proposals.rs new file mode 100644 index 00000000..b4a2b731 --- /dev/null +++ b/crates/server/src/jobs/account_recovery/proposals.rs @@ -0,0 +1,101 @@ +use crate::delta_object::{CosignerSignature, DeltaObject, DeltaStatus}; +use crate::metadata::auth::ChainCosigner; +use crate::network::AppliedState; +use crate::services::proposal_signature::{proposal_tx_summary, verify_proposal_signature}; +use crate::storage::ProposalRecord; +use crate::storage::postgres::recovery::{ProposalChange, ReviewedProposal}; +use miden_protocol::transaction::TransactionSummary; + +/// Reviews an account's pending proposals against the signer set of its restored state. A +/// proposal none of those signers proposed or approved is removed, and a proposal that stays +/// keeps only their approvals. A proposal whose transaction summary cannot be read stays as +/// it is. +pub(super) fn review(restored: &AppliedState, pending: &[ProposalRecord]) -> Vec { + pending + .iter() + .map(|record| ReviewedProposal { + commitment: record.commitment.clone(), + status: record.proposal.status.clone(), + change: change_for(&restored.cosigners, &record.proposal), + }) + .collect() +} + +fn change_for(signers: &[ChainCosigner], proposal: &DeltaObject) -> ProposalChange { + let (timestamp, proposer_id, cosigner_sigs) = match &proposal.status { + DeltaStatus::Pending { + timestamp, + proposer_id, + cosigner_sigs, + } => (timestamp, proposer_id, cosigner_sigs), + DeltaStatus::Candidate { .. } + | DeltaStatus::Canonical { .. } + | DeltaStatus::Retained { .. } + | DeltaStatus::Discarded { .. } => return ProposalChange::Keep, + }; + let Ok(summary) = proposal_tx_summary(&proposal.delta_payload) else { + return ProposalChange::Keep; + }; + let approvals = approvals_of(signers, &summary, cosigner_sigs); + let proposed_by_signer = signers + .iter() + .any(|signer| signer.commitment.eq_ignore_ascii_case(proposer_id)); + if approvals.is_empty() && !proposed_by_signer { + return ProposalChange::Remove; + } + if approvals.len() == cosigner_sigs.len() { + return ProposalChange::Keep; + } + ProposalChange::DropApprovals { + delta_payload: payload_with_approvals(&proposal.delta_payload, &approvals), + status: DeltaStatus::Pending { + timestamp: timestamp.clone(), + proposer_id: proposer_id.clone(), + cosigner_sigs: approvals, + }, + } +} + +fn approvals_of( + signers: &[ChainCosigner], + summary: &TransactionSummary, + signatures: &[CosignerSignature], +) -> Vec { + signatures + .iter() + .filter(|approval| { + verify_proposal_signature(summary, &approval.signature).is_ok_and(|signer| { + signers.iter().any(|known| { + known.commitment.eq_ignore_ascii_case(&signer) + && known.scheme == Some(approval.signature.scheme()) + }) + }) + }) + .cloned() + .collect() +} + +/// The payload with its `signatures` list reduced to the signers of `approvals`. A proposal +/// stores each approval in its status and in its payload, and clients read either. +fn payload_with_approvals( + delta_payload: &serde_json::Value, + approvals: &[CosignerSignature], +) -> serde_json::Value { + let mut payload = delta_payload.clone(); + if let Some(signatures) = payload + .get_mut("signatures") + .and_then(serde_json::Value::as_array_mut) + { + signatures.retain(|entry| { + entry + .get("signer_id") + .and_then(serde_json::Value::as_str) + .is_some_and(|signer| { + approvals + .iter() + .any(|approval| approval.signer_id.eq_ignore_ascii_case(signer)) + }) + }); + } + payload +} diff --git a/crates/server/src/jobs/mod.rs b/crates/server/src/jobs/mod.rs index df1728d8..21477e5a 100644 --- a/crates/server/src/jobs/mod.rs +++ b/crates/server/src/jobs/mod.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "postgres")] +pub(crate) mod account_recovery; pub mod block_watch; pub mod canonicalization; pub mod execution_reconcile; diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index d609e513..405226ff 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -10,10 +10,35 @@ use server::network::{NetworkType, RpcSettings}; use std::env; use std::path::PathBuf; +#[cfg(feature = "postgres")] +use server::builder::handle::StartupRecoveryError; +use server::builder::handle::StartupRecoveryMode; + +const STARTUP_RECOVERY_ENV: &str = "GUARDIAN_STARTUP_ACCOUNT_RECOVERY"; + #[tokio::main] async fn main() { dotenvy::dotenv().ok(); + let startup_recovery = match env::var(STARTUP_RECOVERY_ENV) { + Ok(value) => value + .parse::() + .unwrap_or_else(|error| { + eprintln!("{STARTUP_RECOVERY_ENV} {error}"); + std::process::exit(2); + }), + Err(env::VarError::NotPresent) => StartupRecoveryMode::Off, + Err(env::VarError::NotUnicode(_)) => { + eprintln!("{STARTUP_RECOVERY_ENV} must be valid UTF-8"); + std::process::exit(2); + } + }; + #[cfg(not(feature = "postgres"))] + if startup_recovery != StartupRecoveryMode::Off { + eprintln!("Account recovery requires a Postgres build"); + std::process::exit(2); + } + let keystore_path: PathBuf = env::var("GUARDIAN_KEYSTORE_PATH") .unwrap_or_else(|_| "/var/guardian/keystore".to_string()) .into(); @@ -39,7 +64,7 @@ async fn main() { let release_sweep = server::release_sweep::ReleaseSweepConfig::from_env() .expect("Invalid release sweep configuration"); - ServerBuilder::new() + let server = ServerBuilder::new() .with_logging(LoggingConfig::default()) .network(network_type) .with_rpc(RpcSettings::from_env(network_type).expect("Invalid RPC configuration")) @@ -81,7 +106,24 @@ async fn main() { .cors(cors_layer) .build() .await - .expect("Failed to build server") - .run() - .await; + .expect("Failed to build server"); + + #[cfg(feature = "postgres")] + { + match server.recover_accounts_on_startup(startup_recovery).await { + Ok(()) => {} + Err(error @ StartupRecoveryError::Incomplete(_)) => { + tracing::error!(%error, "Startup account recovery did not finish; serving without it"); + } + Err( + error @ (StartupRecoveryError::LeaseBusy + | StartupRecoveryError::LeaseUnavailable(_)), + ) => { + tracing::error!(%error, "Cannot serve while account recovery may be active elsewhere"); + std::process::exit(1); + } + } + } + + server.run().await; } diff --git a/crates/server/src/schema.rs b/crates/server/src/schema.rs index dd7d0df8..c1e5bda1 100644 --- a/crates/server/src/schema.rs +++ b/crates/server/src/schema.rs @@ -1,5 +1,52 @@ // @generated automatically by Diesel CLI. +diesel::table! { + /// Representation of the `account_recovery_startup` table. + account_recovery_startup (network_config) { + network_config -> Jsonb, + #[max_length = 128] + resume_after -> Varchar, + completed_at -> Nullable, + unresolved_accounts -> Nullable, + pass_unresolved -> Int8, + pass_deferred -> Bool, + } +} + +diesel::table! { + /// Representation of the `account_recovery_pending` table. + account_recovery_pending (account_id) { + #[max_length = 128] + account_id -> Varchar, + outcome -> Text, + reason -> Text, + #[max_length = 128] + stored_commitment -> Varchar, + canonical_nonce -> Nullable, + #[max_length = 128] + canonical_commitment -> Nullable, + history_starts_at_nonce -> Nullable, + #[max_length = 128] + history_base_commitment -> Nullable, + recorded_at -> Timestamptz, + } +} + +diesel::table! { + /// Representation of the `account_recovery_backups` table. + account_recovery_backups (id) { + id -> Int8, + #[max_length = 128] + account_id -> Varchar, + prior_state -> Jsonb, + prior_auth -> Jsonb, + prior_proposals -> Jsonb, + #[max_length = 128] + recovered_commitment -> Varchar, + recovered_at -> Timestamptz, + } +} + diesel::table! { /// Representation of the `states` table. /// diff --git a/crates/server/src/storage/postgres.rs b/crates/server/src/storage/postgres.rs index 5b69e845..6f2d7653 100644 --- a/crates/server/src/storage/postgres.rs +++ b/crates/server/src/storage/postgres.rs @@ -36,6 +36,8 @@ use url::Url; #[path = "postgres_execution.rs"] mod execution; +#[path = "postgres_recovery.rs"] +pub(crate) mod recovery; pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); diff --git a/crates/server/src/storage/postgres_recovery.rs b/crates/server/src/storage/postgres_recovery.rs new file mode 100644 index 00000000..209c94e4 --- /dev/null +++ b/crates/server/src/storage/postgres_recovery.rs @@ -0,0 +1,778 @@ +use std::{sync::Arc, time::Duration}; + +use chrono::{DateTime, Utc}; +use diesel::prelude::*; +use diesel::sql_types::{Array, BigInt, Bool, Jsonb, Nullable, Text, Timestamptz}; +use diesel_async::scoped_futures::{ScopedBoxFuture, ScopedFutureExt}; +use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl, SimpleAsyncConnection}; +use serde_json::Value; +use tokio::time::timeout; + +use crate::coordination::{LeaderElector, Lease, postgres::PgLeaseElector}; +use crate::delta_object::DeltaStatus; +use crate::metadata::{Auth, MidenNetworkType, NetworkConfig}; +use crate::schema::{account_metadata, delta_proposals, states}; +use crate::storage::LeaseFence; +use crate::storage::encryption::{cipher::StorageCipher, envelope::RecordAad}; +use crate::utils::normalize_commitment_hex; + +use super::{Pool, lease_fence_is_current, lock_account_metadata}; + +/// The work in flight that can still land on the account: a delta, or a reserved execution, +/// that starts from `head`, the SQL expression for the account's latest canonical commitment. +fn unsettled_account_work(head: &str) -> String { + format!( + "CASE + WHEN EXISTS ( + SELECT 1 FROM deltas pending WHERE pending.account_id = st.account_id + AND pending.status_kind IN ('candidate', 'retained') + AND pending.prev_commitment = {head} + ) THEN 'a delta awaits its on-chain outcome' + WHEN EXISTS ( + SELECT 1 FROM execution_reservations r + WHERE r.account_id = st.account_id AND r.released_at IS NULL + AND (EXISTS ( + SELECT 1 FROM delta_proposals p + WHERE p.account_id = r.account_id AND p.commitment = r.proposal_id + AND p.prev_commitment = {head} + ) + OR EXISTS ( + SELECT 1 FROM execution_submissions s + WHERE s.account_id = r.account_id AND s.proposal_id = r.proposal_id + AND s.attempt = r.attempt AND s.base_commitment = {head} + )) + ) THEN 'an execution is reserved' + END" + ) +} + +const CANDIDATE_SOURCE: &str = " + FROM account_metadata m + JOIN states st USING (account_id) + LEFT JOIN LATERAL ( + SELECT nonce, new_commitment FROM deltas + WHERE account_id = st.account_id AND status_kind = 'canonical' + ORDER BY nonce DESC LIMIT 1 + ) d ON TRUE + LEFT JOIN LATERAL ( + SELECT nonce, prev_commitment FROM deltas + WHERE account_id = st.account_id AND status_kind = 'canonical' + ORDER BY nonce LIMIT 1 + ) first_held ON TRUE + WHERE m.account_id > $1 AND m.network_config->>'kind' = 'miden' + AND (m.network_config = $2 OR m.network_config->>'network_type' IN ('local', 'devnet')) + AND m.released_at IS NULL + AND (d.nonce IS NOT NULL OR st.updated_at > st.created_at) + AND d.new_commitment IS DISTINCT FROM st.commitment"; + +const HELD_LEASE: &str = " + FROM worker_leases + WHERE lease_name = $1 AND holder_id = $2 AND fence_token = $3 + AND clock_timestamp() < expires_at"; + +#[derive(diesel::QueryableByName)] +pub(crate) struct RecoveryCandidate { + #[diesel(sql_type = Text)] + pub account_id: String, + #[diesel(sql_type = Nullable)] + pub stored_nonce: Option, + #[diesel(sql_type = Text)] + pub stored_commitment: String, + #[diesel(sql_type = Nullable)] + pub canonical_nonce: Option, + #[diesel(sql_type = Nullable)] + pub canonical_commitment: Option, + #[diesel(sql_type = Nullable)] + pub unsettled_work: Option, + #[diesel(sql_type = Nullable)] + pub history_starts_at_nonce: Option, + #[diesel(sql_type = Nullable)] + pub history_base_commitment: Option, + #[diesel(sql_type = Timestamptz)] + state_updated_at: DateTime, + #[diesel(sql_type = BigInt)] + auth_version: i64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PendingOutcome { + Deferred, + Unresolved, +} + +impl PendingOutcome { + fn label(self) -> &'static str { + match self { + Self::Deferred => "deferred", + Self::Unresolved => "unresolved", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ProposalChange { + Keep, + Remove, + DropApprovals { + status: DeltaStatus, + delta_payload: Value, + }, +} + +impl ProposalChange { + pub(crate) fn label(&self) -> &'static str { + match self { + Self::Keep => "keep", + Self::Remove => "remove", + Self::DropApprovals { .. } => "drop_approvals", + } + } +} + +/// A pending proposal as the scan read it, and what the restore does with it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ReviewedProposal { + pub commitment: String, + pub status: DeltaStatus, + pub change: ProposalChange, +} + +/// The review of an account's pending proposals, or the fact that they could not be read. +/// A restore leaves unreadable proposals as they are. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ProposalReview { + Reviewed(Vec), + Unreadable, +} + +/// What a restore writes to one proposal row. +enum ProposalWrite { + Remove, + Replace { status: Value, delta_payload: Value }, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq, diesel::QueryableByName)] +pub(crate) struct ScanProgress { + #[diesel(sql_type = Text)] + pub resume_after: String, + #[diesel(sql_type = BigInt)] + pub unresolved: i64, + #[diesel(sql_type = Bool)] + pub deferred: bool, +} + +pub(crate) struct RecoveryStore { + pool: Pool, + cipher: Option>, + pub leader: Arc, +} + +impl RecoveryStore { + pub(crate) fn new( + pool: Pool, + cipher: Option>, + holder_id: impl Into, + ) -> Self { + let leader = Arc::new(PgLeaseElector::new( + pool.clone(), + "account_recovery", + holder_id, + )); + Self { + pool, + cipher, + leader, + } + } + + pub(crate) async fn startup_completed( + &self, + network_type: MidenNetworkType, + ) -> Result { + #[derive(diesel::QueryableByName)] + struct Completion { + #[diesel(sql_type = Bool)] + completed: bool, + } + + let network = miden_network_config(network_type)?; + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query( + "SELECT EXISTS ( + SELECT 1 FROM account_recovery_startup + WHERE network_config = $1 AND unresolved_accounts IS NOT NULL + ) AS completed", + ) + .bind::(network) + .get_result::(&mut conn) + .await + .map(|row| row.completed) + .map_err(|error| format!("Could not read startup recovery completion: {error}")) + } + + pub(crate) async fn progress( + &self, + network_type: MidenNetworkType, + ) -> Result { + let network = miden_network_config(network_type)?; + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query( + "SELECT resume_after, pass_unresolved AS unresolved, pass_deferred AS deferred + FROM account_recovery_startup WHERE network_config = $1", + ) + .bind::(network) + .get_result::(&mut conn) + .await + .optional() + .map(Option::unwrap_or_default) + .map_err(|error| format!("Could not read startup recovery progress: {error}")) + } + + pub(crate) async fn save_progress( + &self, + network_type: MidenNetworkType, + progress: &ScanProgress, + lease: &Lease, + ) -> Result<(), String> { + let network = miden_network_config(network_type)?; + let written = self + .write("record startup recovery progress", |conn| { + async move { + diesel::sql_query(format!( + "INSERT INTO account_recovery_startup + (network_config, resume_after, pass_unresolved, pass_deferred) + SELECT $4, $5, $6, $7 {HELD_LEASE} + ON CONFLICT (network_config) DO UPDATE + SET resume_after = EXCLUDED.resume_after, + pass_unresolved = EXCLUDED.pass_unresolved, + pass_deferred = EXCLUDED.pass_deferred + WHERE account_recovery_startup.unresolved_accounts IS NULL", + )) + .bind::(&lease.name) + .bind::(&lease.holder_id) + .bind::(lease.fence_token) + .bind::(network) + .bind::(&progress.resume_after) + .bind::(progress.unresolved) + .bind::(progress.deferred) + .execute(conn) + .await + } + .scope_boxed() + }) + .await?; + if written == 0 { + return Err("Account recovery lost its lease before recording scan progress".into()); + } + Ok(()) + } + + pub(crate) async fn complete_startup( + &self, + network_type: MidenNetworkType, + unresolved_accounts: i64, + lease: &Lease, + ) -> Result<(), String> { + let network = miden_network_config(network_type)?; + let written = self.write("record startup recovery completion", |conn| async move { + diesel::sql_query(format!( + "INSERT INTO account_recovery_startup (network_config, unresolved_accounts, completed_at) + SELECT $4, $5, now() {HELD_LEASE} + ON CONFLICT (network_config) DO UPDATE + SET completed_at = EXCLUDED.completed_at, + unresolved_accounts = EXCLUDED.unresolved_accounts, + resume_after = '', + pass_unresolved = 0, + pass_deferred = FALSE + WHERE account_recovery_startup.unresolved_accounts IS NULL", + )) + .bind::(&lease.name) + .bind::(&lease.holder_id) + .bind::(lease.fence_token) + .bind::(network) + .bind::(unresolved_accounts) + .execute(conn) + .await + }.scope_boxed()).await?; + if written == 0 { + return Err( + "Account recovery lost its lease before recording startup completion".into(), + ); + } + Ok(()) + } + + pub(crate) async fn candidates( + &self, + network_type: MidenNetworkType, + after: &str, + limit: i64, + ) -> Result, String> { + let network = miden_network_config(network_type)?; + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query(format!( + "SELECT st.account_id, st.nonce AS stored_nonce, st.commitment AS stored_commitment, + d.nonce AS canonical_nonce, ({unsettled_work}) AS unsettled_work, + d.new_commitment AS canonical_commitment, st.updated_at AS state_updated_at, + m.auth_version, + first_held.nonce AS history_starts_at_nonce, + first_held.prev_commitment AS history_base_commitment + {CANDIDATE_SOURCE} + ORDER BY m.account_id + LIMIT $3", + unsettled_work = unsettled_account_work("d.new_commitment"), + )) + .bind::(after) + .bind::(network) + .bind::(limit) + .load(&mut conn) + .await + .map_err(|error| format!("Could not list recovery candidates: {error}")) + } + + pub(crate) async fn remaining_candidates( + &self, + network_type: MidenNetworkType, + after: &str, + ) -> Result { + #[derive(diesel::QueryableByName)] + struct Remaining { + #[diesel(sql_type = BigInt)] + remaining: i64, + } + + let network = miden_network_config(network_type)?; + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query(format!("SELECT count(*) AS remaining {CANDIDATE_SOURCE}")) + .bind::(after) + .bind::(network) + .get_result::(&mut conn) + .await + .map(|row| row.remaining) + .map_err(|error| format!("Could not count remaining recovery candidates: {error}")) + } + + /// Whether a canonical delta of the account starts from or ends at `commitment`. + pub(crate) async fn holds_canonical_commitment( + &self, + account_id: &str, + commitment: &str, + ) -> Result { + self.holds_delta( + account_id, + commitment, + "status_kind = 'canonical' AND (new_commitment = $2 OR prev_commitment = $2)", + ) + .await + } + + /// Whether a delta of the account that still awaits its on-chain outcome ends at + /// `commitment`. + pub(crate) async fn holds_landed_delta( + &self, + account_id: &str, + commitment: &str, + ) -> Result { + self.holds_delta( + account_id, + commitment, + "status_kind IN ('candidate', 'retained') AND new_commitment = $2", + ) + .await + } + + async fn holds_delta( + &self, + account_id: &str, + commitment: &str, + condition: &str, + ) -> Result { + #[derive(diesel::QueryableByName)] + struct Held { + #[diesel(sql_type = Bool)] + held: bool, + } + + let mut conn = self.pool.get().await.map_err(|error| error.to_string())?; + diesel::sql_query(format!( + "SELECT EXISTS ( + SELECT 1 FROM deltas WHERE account_id = $1 AND {condition} + ) AS held", + )) + .bind::(account_id) + .bind::(commitment) + .get_result::(&mut conn) + .await + .map(|row| row.held) + .map_err(|error| format!("Could not read the account's deltas: {error}")) + } + + pub(crate) async fn record_pending( + &self, + candidate: &RecoveryCandidate, + outcome: PendingOutcome, + reason: &str, + lease: &Lease, + ) -> Result<(), String> { + let written = self + .write("record the pending account", |conn| { + async move { + diesel::sql_query(format!( + "INSERT INTO account_recovery_pending + (account_id, outcome, reason, stored_commitment, canonical_nonce, + canonical_commitment, history_starts_at_nonce, history_base_commitment) + SELECT $4, $5, $6, $7, $8, $9, $10, $11 {HELD_LEASE} + ON CONFLICT (account_id) DO UPDATE + SET outcome = EXCLUDED.outcome, + reason = EXCLUDED.reason, + stored_commitment = EXCLUDED.stored_commitment, + canonical_nonce = EXCLUDED.canonical_nonce, + canonical_commitment = EXCLUDED.canonical_commitment, + history_starts_at_nonce = EXCLUDED.history_starts_at_nonce, + history_base_commitment = EXCLUDED.history_base_commitment, + recorded_at = EXCLUDED.recorded_at", + )) + .bind::(&lease.name) + .bind::(&lease.holder_id) + .bind::(lease.fence_token) + .bind::(&candidate.account_id) + .bind::(outcome.label()) + .bind::(reason) + .bind::(&candidate.stored_commitment) + .bind::, _>(candidate.canonical_nonce) + .bind::, _>(candidate.canonical_commitment.as_deref()) + .bind::, _>(candidate.history_starts_at_nonce) + .bind::, _>(candidate.history_base_commitment.as_deref()) + .execute(conn) + .await + } + .scope_boxed() + }) + .await?; + if written == 0 { + return Err( + "Account recovery lost its lease before recording the pending account".into(), + ); + } + Ok(()) + } + + pub(crate) async fn clear_pending( + &self, + account_id: &str, + lease: &Lease, + ) -> Result<(), String> { + self.write("clear the pending account", |conn| { + async move { + diesel::sql_query(format!( + "DELETE FROM account_recovery_pending WHERE account_id = $4 + AND EXISTS (SELECT 1 {HELD_LEASE})", + )) + .bind::(&lease.name) + .bind::(&lease.holder_id) + .bind::(lease.fence_token) + .bind::(account_id) + .execute(conn) + .await + .map(drop) + } + .scope_boxed() + }) + .await + } + + pub(crate) async fn apply( + &self, + candidate: &RecoveryCandidate, + state_json: &Value, + auth: &Auth, + proposals: &ProposalReview, + now: DateTime, + lease: &Lease, + ) -> Result { + let account_id = &candidate.account_id; + let reviewed = match proposals { + ProposalReview::Reviewed(reviewed) => Some(reviewed), + ProposalReview::Unreadable => None, + }; + let expected = reviewed.map(|reviewed| { + let mut expected: Vec<(String, DeltaStatus)> = reviewed + .iter() + .filter(|proposal| proposal.status.is_pending()) + .map(|proposal| (proposal.commitment.clone(), proposal.status.clone())) + .collect(); + expected.sort_by(|left, right| left.0.cmp(&right.0)); + expected + }); + let writes = reviewed + .into_iter() + .flatten() + .filter_map(|proposal| { + self.proposal_write(account_id, proposal) + .map(|write| write.map(|write| (proposal.commitment.clone(), write))) + .transpose() + }) + .collect::, _>>()?; + let changed: Vec = writes + .iter() + .map(|(commitment, _)| commitment.clone()) + .collect(); + let canonical_nonce = candidate + .canonical_nonce + .ok_or("Recovery target has no canonical nonce")?; + let canonical_commitment = candidate + .canonical_commitment + .as_deref() + .ok_or("Recovery target has no canonical commitment")?; + let stored_json = match &self.cipher { + Some(cipher) => cipher + .encrypt(&RecordAad::State { account_id }, state_json) + .map_err(|error| error.to_string())?, + None => state_json.clone(), + }; + let auth_json = serde_json::to_value(auth).map_err(|error| error.to_string())?; + self.write("restore the account", |conn| { + async move { + if lock_account_metadata(conn, account_id) + .await + .optional()? + .is_none() + || !candidate + .lock_if_eligible(conn, canonical_nonce, canonical_commitment) + .await? + { + return Ok(false); + } + + let fence = LeaseFence { + lease_name: lease.name.clone(), + holder_id: lease.holder_id.clone(), + fence_token: lease.fence_token, + }; + if !lease_fence_is_current(conn, &fence).await? { + return Ok(false); + } + if let Some(expected) = &expected + && pending_proposals(conn, account_id).await? != *expected + { + return Ok(false); + } + + diesel::sql_query( + "INSERT INTO account_recovery_backups + (account_id, prior_state, prior_auth, prior_proposals, + recovered_commitment) + SELECT st.account_id, to_jsonb(st), m.auth, + COALESCE(( + SELECT jsonb_agg(to_jsonb(p) ORDER BY p.commitment) + FROM delta_proposals p + WHERE p.account_id = st.account_id AND p.commitment = ANY($3) + ), '[]'::jsonb), + $2 + FROM states st JOIN account_metadata m USING (account_id) + WHERE st.account_id = $1", + ) + .bind::(account_id) + .bind::(canonical_commitment) + .bind::, _>(&changed) + .execute(conn) + .await?; + + for (commitment, write) in &writes { + let proposal = delta_proposals::table + .filter(delta_proposals::account_id.eq(account_id)) + .filter(delta_proposals::commitment.eq(commitment)); + match write { + ProposalWrite::Replace { + status, + delta_payload, + } => { + diesel::update(proposal) + .set(( + delta_proposals::status.eq(status), + delta_proposals::delta_payload.eq(delta_payload), + )) + .execute(conn) + .await? + } + ProposalWrite::Remove => diesel::delete(proposal).execute(conn).await?, + }; + } + + let written_nonce = diesel::update(states::table.find(account_id)) + .set(( + states::commitment.eq(canonical_commitment), + states::state_json.eq(stored_json), + states::nonce.eq(Some(canonical_nonce)), + states::updated_at.eq(now), + )) + .returning(states::nonce) + .get_result::>(conn) + .await?; + if written_nonce.is_none() { + diesel::update(states::table.find(account_id)) + .set(states::nonce.eq(Some(canonical_nonce))) + .execute(conn) + .await?; + } + + diesel::update(account_metadata::table.find(account_id)) + .set(( + account_metadata::auth.eq(auth_json), + account_metadata::updated_at.eq(now), + )) + .execute(conn) + .await?; + diesel::sql_query("DELETE FROM account_recovery_pending WHERE account_id = $1") + .bind::(account_id) + .execute(conn) + .await?; + Ok(true) + } + .scope_boxed() + }) + .await + } + + /// The row write for a reviewed proposal, with its payload encrypted as the storage layer + /// stores it. + fn proposal_write( + &self, + account_id: &str, + proposal: &ReviewedProposal, + ) -> Result, String> { + let (status, delta_payload) = match &proposal.change { + ProposalChange::Keep => return Ok(None), + ProposalChange::Remove => return Ok(Some(ProposalWrite::Remove)), + ProposalChange::DropApprovals { + status, + delta_payload, + } => (status, delta_payload), + }; + let commitment = normalize_commitment_hex(&proposal.commitment) + .unwrap_or_else(|_| proposal.commitment.clone()); + let delta_payload = match &self.cipher { + Some(cipher) => cipher + .encrypt( + &RecordAad::Proposal { + account_id, + commitment: &commitment, + }, + delta_payload, + ) + .map_err(|error| error.to_string())?, + None => delta_payload.clone(), + }; + Ok(Some(ProposalWrite::Replace { + status: serde_json::to_value(status).map_err(|error| error.to_string())?, + delta_payload, + })) + } + + async fn write<'a, T, F>(&self, operation: &str, work: F) -> Result + where + T: Send + 'a, + F: for<'conn> FnOnce( + &'conn mut AsyncPgConnection, + ) -> ScopedBoxFuture<'a, 'conn, QueryResult> + + Send + + 'a, + { + let mut conn = timeout(Duration::from_secs(3), self.pool.get()) + .await + .map_err(|_| format!("Timed out waiting for a connection to {operation}"))? + .map_err(|error| format!("Could not get a connection to {operation}: {error}"))?; + AsyncConnection::transaction(&mut *conn, |conn| { + async move { + conn.batch_execute( + "SET LOCAL statement_timeout = '2s'; SET LOCAL lock_timeout = '2s';", + ) + .await?; + work(conn).await + } + .scope_boxed() + }) + .await + .map_err(|error| format!("Could not {operation}: {error}")) + } +} + +/// The account's pending proposals and their statuses, locked for the transaction and read +/// the way the storage layer reads them. A row whose status cannot be read is left out, as it +/// is from the review this is compared with. +async fn pending_proposals( + conn: &mut AsyncPgConnection, + account_id: &str, +) -> QueryResult> { + #[derive(diesel::QueryableByName)] + struct PendingProposal { + #[diesel(sql_type = Text)] + commitment: String, + #[diesel(sql_type = Jsonb)] + status: Value, + } + + let mut pending: Vec<(String, DeltaStatus)> = diesel::sql_query( + "SELECT commitment, status FROM delta_proposals + WHERE account_id = $1 AND status_kind = 'pending' + FOR UPDATE", + ) + .bind::(account_id) + .load::(conn) + .await? + .into_iter() + .filter_map(|proposal| { + serde_json::from_value::(proposal.status) + .ok() + .filter(DeltaStatus::is_pending) + .map(|status| (proposal.commitment, status)) + }) + .collect(); + pending.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(pending) +} + +fn miden_network_config(network_type: MidenNetworkType) -> Result { + serde_json::to_value(NetworkConfig::Miden { network_type }).map_err(|error| error.to_string()) +} + +#[derive(diesel::QueryableByName)] +struct LockedState { + #[diesel(sql_type = Text, column_name = account_id)] + _account_id: String, +} + +impl RecoveryCandidate { + async fn lock_if_eligible( + &self, + conn: &mut AsyncPgConnection, + canonical_nonce: i64, + canonical_commitment: &str, + ) -> QueryResult { + diesel::sql_query(format!( + "SELECT st.account_id + FROM states st JOIN account_metadata m USING (account_id) + WHERE st.account_id = $1 AND st.nonce IS NOT DISTINCT FROM $2 AND st.commitment = $3 + AND st.updated_at = $4 AND m.auth_version = $5 + AND m.released_at IS NULL AND ({unsettled_work}) IS NULL + AND ($6, $7) = ( + SELECT d.nonce, d.new_commitment FROM deltas d + WHERE d.account_id = st.account_id AND d.status_kind = 'canonical' + ORDER BY d.nonce DESC LIMIT 1 + ) + FOR UPDATE OF st", + unsettled_work = unsettled_account_work("$7"), + )) + .bind::(&self.account_id) + .bind::, _>(self.stored_nonce) + .bind::(&self.stored_commitment) + .bind::(self.state_updated_at) + .bind::(self.auth_version) + .bind::(canonical_nonce) + .bind::(canonical_commitment) + .get_result::(conn) + .await + .optional() + .map(|row| row.is_some()) + } +} diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 39fc8a0e..d1d71650 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -30,6 +30,7 @@ the runtime env vars in this document. | `GUARDIAN_KEYSTORE_PATH` | `/var/guardian/keystore` | any | Local Falcon/ECDSA key files (ACK signers and per-account creds). | | `GUARDIAN_DB_POOL_MAX_SIZE` | `16`; `32` when `GUARDIAN_ENV=prod` | `postgres` | Storage backend pool size. | | `GUARDIAN_METADATA_DB_POOL_MAX_SIZE` | matches storage | `postgres` | Metadata backend pool size; usually leave equal. | +| `GUARDIAN_STARTUP_ACCOUNT_RECOVERY` | `off` | `postgres` | `off`, `dry-run` or `apply`. Checks account records against the Miden network for up to 5 minutes before starting API listeners. `dry-run` logs what it would change and writes nothing. `apply` continues on later startups where it stopped; use it with stop-before-start upgrades and one replica, and Guardian skips it when `GUARDIAN_MAX_REPLICAS` exceeds `1`. In `apply`, if another process holds the recovery lease, or the server cannot confirm it holds it alone, the server exits before opening its listeners. AWS deployments set it through Terraform variable `guardian_startup_account_recovery`, which also lengthens the task health check timings. See [Startup account recovery](./PRODUCTION.md#startup-account-recovery). | | `GUARDIAN_METADATA_CACHE_MAX_BYTES` | `268435456` (256 MiB) | `postgres` | Memory bound of the per-replica account metadata cache used by read routes. Every read still confirms the copy is current against the database (one indexed lookup of a version the database raises on every metadata update), so the cache never serves stale authorization data; it only saves loading and parsing the row. `0` disables caching. | | `GUARDIAN_CANONICALIZATION_FAST_PROMOTION_ENABLED` | `true` | any | Enables the additional promotion-only pass for recent candidates. Set to `false` to use only the full canonicalization interval; AWS deployments can set Terraform variable `guardian_canonicalization_fast_promotion_enabled = false`. | | `GUARDIAN_CANONICALIZATION_MAX_CONCURRENT_ACCOUNTS` | `10`; `50` when `GUARDIAN_ENV=prod` | any | Accounts one canonicalization pass processes in parallel; `1` = fully sequential. Each account holds a DB connection only during its short fenced transactions (the dominant cost is a connectionless chain RPC), so this may exceed `GUARDIAN_DB_POOL_MAX_SIZE`; simultaneous write bursts just queue briefly at the pool. | diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index d087315a..f03c7862 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -350,6 +350,191 @@ Two Guardian-specific caveats when restoring from a backup: The verification and restore procedure is in [`runbooks/backup-restore.md`](./runbooks/backup-restore.md). +### Startup account recovery + +Guardian can check its stored Miden account records against the Miden network +when it starts, and bring a record back in line with the account's canonical +history when the two disagree. `GUARDIAN_STARTUP_ACCOUNT_RECOVERY` selects +the mode: `off` (the default), `dry-run` or `apply`. It needs a Guardian +backed by Postgres. With the bundled Compose files, set it in `.env`. + +**Previewing with a dry run** + +Start Guardian once with `GUARDIAN_STARTUP_ACCOUNT_RECOVERY=dry-run` first. A +dry run performs the same checks, logs what `apply` would do with each +account, and writes nothing: no account record, no report row and no +progress. It takes no lease, so it runs with any number of replicas, and it +checks from the first account each time. It names each proposal a restore +would change, and says so when `apply` would not run here (more than one +replica, or the check already completed). The final log line counts the +accounts that would be restored (`restorable`), deferred (`retryable`) and +left unresolved (`unresolved`). A dry run stops after 5 minutes like `apply`; +that line then shows `stopped_early` and how many accounts it did not reach +(`remaining`). + +**Applying** + +- Set `GUARDIAN_STARTUP_ACCOUNT_RECOVERY=apply` on a single Guardian instance. +- Stop the old Guardian process before the new one starts. Rolling + deployments overlap old and new servers even with a target of one instance, + so do not use `apply` for those. +- Guardian skips `apply` when `GUARDIAN_MAX_REPLICAS` exceeds `1`. +- On the AWS stack in `infra/`, set Terraform variable + `guardian_startup_account_recovery` to `"dry-run"` or `"apply"` for that one + deploy. It passes the mode to the task and lengthens the container health + check start period and the load balancer grace period, so the task is not + replaced while its listeners are closed. For `apply`, running a single task + and keeping older tasks from running during the deploy remain manual steps. + Terraform sets the task count and autoscaling, so make them through its + variables: first apply `server_autoscaling_enabled = false` with + `server_desired_count = 0` and wait for the tasks to stop, then apply + `guardian_startup_account_recovery = "apply"` with + `server_desired_count = 1`. Leave `guardian_max_replicas` unset or at `1` + for that deploy. Stopping a task or scaling the service outside + Terraform is not enough, because the service or the next deploy starts a + replacement. Once the check has completed, set the mode back to `"off"` + and restore the usual task count and autoscaling. + +**What it does** + +The check runs before the API listeners open. For each account whose stored +record differs from its latest canonical delta, Guardian asks the configured +Miden node for the account's current commitment. It covers accounts recorded +for the configured network and accounts recorded as `local` or `devnet`, which +is how an account registered without a network is stored. + +- If the stored record already matches the network, nothing changes. +- If the configured node has no record of the account and Guardian holds no + canonical deltas for it, the stored record stands only when it is the + account's first state. Any other record is listed as unresolved. +- If the network is past the canonical deltas Guardian holds and no delta in + flight ends at the network's state, the account is listed as unresolved. If + the configured node reports an earlier state of the account, the account is + deferred. +- If Guardian holds the account's canonical deltas from creation, it rebuilds + the account from them. An account whose history takes more than a minute to + rebuild is listed as unresolved. When the result matches the network and + is bound to this server's ACK key, Guardian stores it together with its + nonce and the account metadata derived from it. The same transaction saves + the previous record in `account_recovery_backups`; encrypted state keeps its + original ciphertext. +- Otherwise the account is left unchanged and listed in + `account_recovery_pending`. + +A restore also reviews the account's pending proposals against the signers of +the restored state. A proposal that one of those signers proposed or approved +stays, keeping only their approvals. Any other pending proposal is removed, +except one whose transaction summary cannot be read, which stays as it is. +Every proposal a restore changes or removes is saved with the previous record +in `account_recovery_backups.prior_proposals`. If the account's proposals +cannot be read, the restore leaves them as they are. + +An account with work in flight (a delta awaiting its on-chain outcome or a +reserved execution) is deferred and checked again on a later startup. Work +counts when it starts from the account's latest canonical state, or when it is +a delta that ends at the network's current state. A paused account is checked +like any other, and stays paused. + +**How long it takes** + +- Guardian checks accounts for up to 5 minutes and lets a write in progress + finish. The listeners can stay closed for about 6 minutes: the 5-minute + budget, up to 35 seconds waiting for the recovery lease, and the final + writes. Allow for that in any startup health check or grace period. +- The check stops for this startup after three account checks in a row fail, + for example while the Miden node is unreachable. +- In `apply`, Guardian records where it stopped in + `account_recovery_startup.resume_after`, and the next startup in `apply` + continues from there. +- The check completes when a pass reaches the last account without deferring + any. A pass can span several startups: one that stops early keeps its + position and its counts. A pass that deferred an account is followed by a + new pass from the first account. On completion Guardian records how many + accounts it could not bring up to date in + `account_recovery_startup.unresolved_accounts`, and later startups skip the + check. Until then, each startup in `apply` runs it again. +- An account Guardian holds canonical deltas for, but the configured node has + no record of, is deferred. It keeps the check open until the node knows the + account or the account is released. + +**If it cannot run** + +In `apply`, Guardian exits before opening its listeners if another process +holds the recovery lease, or if it cannot confirm it holds the lease alone. It +first waits up to 35 seconds for a lease a stopped process left behind. Any +other error, and any error in a dry run, is logged, and Guardian opens its +listeners without finishing the check. + +**Reviewing the result** + +After `apply`, `account_recovery_pending` has one row for each account that +was checked and not brought up to date. `outcome` is `deferred` (a later +startup checks it again) or `unresolved` (Guardian does not hold what it +needs), and `reason` says why. When the canonical history held here does not +reach back to the account's creation, `history_starts_at_nonce` and +`history_base_commitment` show where it begins. + +An unresolved account the network has a record of is brought up to date the +next time it is registered with its current state. One the network has no +record of cannot be registered again while its stored state remains. Remove +that row with the statement below and keep its output, which is the removed +row. It removes the row only while it still holds the `stored_commitment` +listed for the account in `account_recovery_pending`. The account's next +registration with its first state is then accepted: + +```sql +DELETE FROM states +WHERE account_id = '' AND commitment = '' +RETURNING to_jsonb(states); +``` + +A row is removed once its account is brought up to date or found unchanged. A +row whose `stored_commitment` no longer matches the account's stored state is +out of date. Export the current list with: + +```sql +SELECT jsonb_agg(to_jsonb(p) ORDER BY p.account_id) +FROM account_recovery_pending p +JOIN states st ON st.account_id = p.account_id + AND st.commitment = p.stored_commitment; +``` + +To put back the record a restore replaced, use its row in +`account_recovery_backups`. The first statement changes the state only while +the account still holds the restored commitment: + +```sql +BEGIN; +UPDATE states st +SET state_json = b.prior_state->'state_json', + commitment = b.prior_state->>'commitment', + nonce = (b.prior_state->>'nonce')::bigint, + updated_at = now() +FROM account_recovery_backups b +WHERE b.id = AND st.account_id = b.account_id + AND st.commitment = b.recovered_commitment; +UPDATE account_metadata m SET auth = b.prior_auth, updated_at = now() +FROM account_recovery_backups b +WHERE b.id = AND m.account_id = b.account_id; +INSERT INTO delta_proposals +SELECT p.* FROM account_recovery_backups b, + jsonb_populate_recordset(NULL::delta_proposals, b.prior_proposals) p +WHERE b.id = +ON CONFLICT (account_id, commitment) DO UPDATE + SET status = EXCLUDED.status, status_kind = EXCLUDED.status_kind, + status_timestamp = EXCLUDED.status_timestamp, + delta_payload = EXCLUDED.delta_payload; +COMMIT; +``` + +To run the check again after it completed, remove its completion record and +restart Guardian: + +```sql +DELETE FROM account_recovery_startup +WHERE network_config->>'network_type' = ''; +``` + ## Storage encryption Guardian can encrypt the sensitive stored payloads (account state, delta and diff --git a/infra/README.md b/infra/README.md index b8e22f28..3eefaac7 100644 --- a/infra/README.md +++ b/infra/README.md @@ -116,6 +116,7 @@ server_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/guardian-server # guardian_canonicalization_fast_promotion_enabled = false # guardian_max_pending_candidates_per_account = 4 # opt in to queueing chained candidates (server default 1, max 16; upgrade TS clients first, see docs/CONFIGURATION.md) # guardian_release_sweep_enabled = false # turns the release sweep's walk of the fleet off +# guardian_startup_account_recovery = "dry-run" # then "apply" for one deploy: single task, service scaled to zero first; see docs/PRODUCTION.md # guardian_block_watch_enabled = false # kill switch for the committed-block feed # Optional: dashboard operator Falcon public keys managed by Terraform @@ -303,6 +304,7 @@ aws ecr delete-repository --repository-name "$ECR_REPO_NAME" --force --region "$ | `guardian_canonicalization_fast_promotion_enabled` | `true` | Enables the recent-candidate promotion-only pass in the ECS task definition | | `guardian_max_pending_candidates_per_account` | server default (`1`) | `GUARDIAN_MAX_PENDING_CANDIDATES_PER_ACCOUNT`: chained candidate deltas one account may hold in flight (issue #17). `1` keeps one in-flight candidate per account; `2` to `16` opts in to queueing back-to-back transactions, after TypeScript clients are upgraded (see [`docs/CONFIGURATION.md`](../docs/CONFIGURATION.md)) | | `guardian_release_sweep_enabled` | `true` | Runs the release sweep's walk of the fleet (the rotation); `false` turns the walk off, and block watch too unless `guardian_block_watch_enabled` is set | +| `guardian_startup_account_recovery` | `off` | `GUARDIAN_STARTUP_ACCOUNT_RECOVERY`: `off`, `dry-run` or `apply`. Checks account records before the listeners open; `dry-run` reports what it would change and writes nothing. Any value but `off` lengthens the container health check start period (to 300 s, with five retries) and the load balancer grace period (to 420 s) so the task is not replaced during the check. `apply` needs a single task and a deploy with no older task still running; set it back to `off` once the check has completed. See [Startup account recovery](../docs/PRODUCTION.md#startup-account-recovery) | | `guardian_release_sweep_rotation_seconds` | server default (`21600`) | Target time for one release sweep walk of the fleet; `604800` (weekly) is enough once block watch is healthy | | `guardian_block_watch_enabled` | unset (follows `guardian_release_sweep_enabled`) | Runs block watch (issue #585): switch detection from the node's committed-block feed; `false` is its kill switch, `true` runs it even with the release sweep's walk turned off | | `guardian_block_watch_transport` | `auto` | `GUARDIAN_BLOCK_WATCH_TRANSPORT`: `auto` (the block stream, polling when it is unavailable), `stream`, or `poll` | diff --git a/infra/data.tf b/infra/data.tf index e78c6200..f16836b1 100644 --- a/infra/data.tf +++ b/infra/data.tf @@ -166,6 +166,15 @@ locals { (local.effective_server_memory - local.fargate_memory_by_cpu[tostring(local.effective_server_cpu)].min) % local.fargate_memory_by_cpu[tostring(local.effective_server_cpu)].step == 0, false ) + # Account recovery keeps the listeners closed for about 6 minutes at most: + # a 5-minute scan budget, up to 35 seconds waiting for a lease a stopped + # task left behind, and the writes in progress when the budget ends. 300 is + # the longest container start period ECS accepts; five health check retries + # at 30 seconds cover the remainder with a margin. + server_health_check_start_period_seconds = var.guardian_startup_account_recovery == "off" ? 10 : 300 + server_health_check_retries = var.guardian_startup_account_recovery == "off" ? 3 : 5 + server_health_check_grace_period_seconds = var.guardian_startup_account_recovery == "off" ? 30 : 420 + effective_server_desired_count = var.server_desired_count != null ? var.server_desired_count : (local.is_prod ? 2 : 1) effective_server_autoscaling_enabled = var.server_autoscaling_enabled != null ? var.server_autoscaling_enabled : local.is_prod effective_server_autoscaling_min_capacity = var.server_autoscaling_min_capacity != null ? var.server_autoscaling_min_capacity : local.effective_server_desired_count diff --git a/infra/ecs.tf b/infra/ecs.tf index ec42176f..12c1bbd5 100644 --- a/infra/ecs.tf +++ b/infra/ecs.tf @@ -202,6 +202,10 @@ resource "aws_ecs_task_definition" "server" { name = "GUARDIAN_RELEASE_SWEEP_ENABLED" value = tostring(var.guardian_release_sweep_enabled) }, + { + name = "GUARDIAN_STARTUP_ACCOUNT_RECOVERY" + value = var.guardian_startup_account_recovery + }, { name = "GUARDIAN_BLOCK_WATCH_TRANSPORT" value = var.guardian_block_watch_transport @@ -421,8 +425,8 @@ resource "aws_ecs_task_definition" "server" { command = ["CMD", "/healthcheck"] interval = 30 timeout = 5 - retries = 3 - startPeriod = 10 + retries = local.server_health_check_retries + startPeriod = local.server_health_check_start_period_seconds } logConfiguration = { @@ -460,7 +464,7 @@ resource "aws_ecs_service" "server" { } } - health_check_grace_period_seconds = 30 + health_check_grace_period_seconds = local.server_health_check_grace_period_seconds network_configuration { subnets = local.subnet_ids diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 2a31543e..e902f607 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -130,6 +130,7 @@ server_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/guardian-server # guardian_canonicalization_fast_promotion_enabled = false # guardian_max_pending_candidates_per_account = 4 # opt in to queueing chained candidates per account (server default 1, max 16) # guardian_release_sweep_enabled = false # turns the release sweep's walk of the fleet off +# guardian_startup_account_recovery = "dry-run" # then "apply" for one deploy: single task, service scaled to zero first; see docs/PRODUCTION.md # guardian_release_sweep_rotation_seconds = 21600 # 604800 (weekly) once block watch is healthy # guardian_release_sweep_max_rate_per_second = 5 # guardian_block_watch_enabled = false # kill switch for the committed-block feed (issue #585); unset follows guardian_release_sweep_enabled diff --git a/infra/variables.tf b/infra/variables.tf index 75ac8df0..2aaebe4f 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -570,6 +570,29 @@ variable "guardian_release_sweep_enabled" { default = true } +variable "guardian_startup_account_recovery" { + description = <<-EOT + Whether the server checks its account records before it opens its + listeners (GUARDIAN_STARTUP_ACCOUNT_RECOVERY): off, dry-run (reports what + it would change and writes nothing) or apply. Any value but off also + lengthens the container health check start period and the load balancer + grace period, so a task is not replaced while the check keeps the + listeners closed. For apply the operator arranges the rest: the server + skips the check unless it runs as a single task, and the deploy must not + overlap an older task. Before the deploy that turns this on, apply + server_autoscaling_enabled = false with server_desired_count = 0 and wait + for the tasks to stop, then deploy with server_desired_count = 1. Set it + back to off once the check has completed. + EOT + type = string + default = "off" + + validation { + condition = contains(["off", "dry-run", "apply"], var.guardian_startup_account_recovery) + error_message = "guardian_startup_account_recovery must be off, dry-run or apply." + } +} + variable "guardian_block_watch_enabled" { description = <<-EOT Whether block watch runs (GUARDIAN_BLOCK_WATCH_ENABLED): the committed-block