Skip to content
Merged
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DROP TABLE account_recovery_backups;
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
CREATE TABLE account_recovery_backups (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
account_id VARCHAR(128) NOT NULL,
prior_state JSONB NOT NULL,
prior_auth JSONB NOT NULL,
prior_proposals JSONB NOT NULL DEFAULT '[]'::jsonb,
recovered_commitment VARCHAR(128) NOT NULL,
recovered_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DROP TABLE account_recovery_startup;
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
CREATE TABLE account_recovery_startup (
network_config JSONB PRIMARY KEY,
resume_after VARCHAR(128) NOT NULL DEFAULT '',
completed_at TIMESTAMPTZ,
unresolved_accounts BIGINT,
pass_unresolved BIGINT NOT NULL DEFAULT 0,
pass_deferred BOOLEAN NOT NULL DEFAULT FALSE
);
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DROP TABLE account_recovery_pending;
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
CREATE TABLE account_recovery_pending (
account_id VARCHAR(128) PRIMARY KEY,
outcome TEXT NOT NULL CHECK (outcome IN ('deferred', 'unresolved')),
reason TEXT NOT NULL,
stored_commitment VARCHAR(128) NOT NULL,
canonical_nonce BIGINT,
canonical_commitment VARCHAR(128),
history_starts_at_nonce BIGINT,
history_base_commitment VARCHAR(128),
recorded_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
102 changes: 102 additions & 0 deletions crates/server/src/builder/handle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ pub struct ServerHandle {
pub(crate) stats_leader: std::sync::Arc<dyn crate::coordination::LeaderElector>,
/// Single-owner lease for the chain-driven release sweep (issue #434).
pub(crate) release_sweep_leader: std::sync::Arc<dyn crate::coordination::LeaderElector>,
#[cfg(feature = "postgres")]
pub(crate) account_recovery: Option<(
std::sync::Arc<crate::storage::postgres::recovery::RecoveryStore>,
crate::metadata::MidenNetworkType,
)>,
pub(crate) startup_info: StartupInfo,
pub(crate) cors_layer: Option<CorsLayer>,
pub(crate) rate_limit_config: Option<RateLimitConfig>,
Expand All @@ -65,7 +70,89 @@ pub struct ServerHandle {
pub(crate) grpc_port: u16,
}

/// Whether the server checks its account records at startup, and whether that check writes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StartupRecoveryMode {
Off,
DryRun,
Apply,
}

impl std::str::FromStr for StartupRecoveryMode {
type Err = String;

fn from_str(value: &str) -> Result<Self, Self::Err> {
match value.trim() {
"off" => Ok(Self::Off),
"dry-run" => Ok(Self::DryRun),
"apply" => Ok(Self::Apply),
other => Err(format!("must be off, dry-run or apply, got {other:?}")),
}
}
}

#[cfg(feature = "postgres")]
#[derive(Debug, thiserror::Error)]
pub enum StartupRecoveryError {
#[error("Another process already holds the account recovery lease")]
LeaseBusy,
#[error("Could not establish exclusive account recovery: {0}")]
LeaseUnavailable(String),
#[error("{0}")]
Incomplete(String),
}

#[cfg(feature = "postgres")]
impl From<String> for StartupRecoveryError {
fn from(reason: String) -> Self {
Self::Incomplete(reason)
}
}

impl ServerHandle {
/// Recover accounts once before a stop-before-start deployment opens its listeners.
#[cfg(feature = "postgres")]
pub async fn recover_accounts_on_startup(
&self,
mode: StartupRecoveryMode,
) -> Result<(), StartupRecoveryError> {
if mode == StartupRecoveryMode::Off {
return Ok(());
}
let (store, network_type) = self
.account_recovery
.as_ref()
.ok_or_else(|| "Account recovery requires shared Postgres storage".to_owned())?;
let single_replica = crate::middleware::rate_limit::max_replicas_from_env()? == 1;
let canonicalizes = self.app_state.canonicalization.is_some();
match mode {
StartupRecoveryMode::Off => Ok(()),
StartupRecoveryMode::DryRun => {
if !single_replica {
tracing::warn!(
"Dry run: apply would be skipped because GUARDIAN_MAX_REPLICAS exceeds 1"
);
}
if !canonicalizes {
tracing::warn!("Dry run: apply would not run without canonicalization");
}
crate::jobs::account_recovery::dry_run(&self.app_state, store, *network_type).await
}
StartupRecoveryMode::Apply if !single_replica => {
tracing::warn!(
"Skipping startup account recovery because GUARDIAN_MAX_REPLICAS exceeds 1"
);
Ok(())
}
StartupRecoveryMode::Apply if !canonicalizes => Err(StartupRecoveryError::Incomplete(
"Account recovery requires canonicalization".into(),
)),
StartupRecoveryMode::Apply => {
crate::jobs::account_recovery::run_once(&self.app_state, store, *network_type).await
}
}
}

/// Run the server with the configured settings
pub async fn run(self) {
self.startup_info.log();
Expand Down Expand Up @@ -821,3 +908,18 @@ fn start_session_sweep_worker(state: AppState) {
}
});
}

#[cfg(test)]
mod startup_recovery_mode_tests {
use super::StartupRecoveryMode;

#[test]
fn the_mode_has_three_values_and_takes_no_boolean() {
assert_eq!("off".parse(), Ok(StartupRecoveryMode::Off));
assert_eq!(" dry-run ".parse(), Ok(StartupRecoveryMode::DryRun));
assert_eq!("apply".parse(), Ok(StartupRecoveryMode::Apply));
for rejected in ["true", "false", "", "Apply"] {
assert!(rejected.parse::<StartupRecoveryMode>().is_err());
}
}
}
7 changes: 7 additions & 0 deletions crates/server/src/builder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,11 @@ impl ServerBuilder {

let ack = self.ack.ok_or("AckRegistry not set. Use .ack(...)")?;
let coordination = self.coordination;
#[cfg(feature = "postgres")]
let account_recovery = coordination
.as_ref()
.and_then(|handles| handles.account_recovery.clone())
.map(|store| (store, crate::metadata::MidenNetworkType::from(network_type)));
// Fail closed before anything else: the Postgres backend must never fall
// back to per-process coordination (AlwaysLeader + in-memory sessions),
// which would let every replica run canonicalization and split auth
Expand Down Expand Up @@ -790,6 +795,8 @@ impl ServerBuilder {
leader,
stats_leader,
release_sweep_leader,
#[cfg(feature = "postgres")]
account_recovery,
startup_info,
cors_layer: self.cors_layer,
rate_limit_config: Some(rate_limit_config),
Expand Down
13 changes: 12 additions & 1 deletion crates/server/src/coordination/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ pub struct CoordinationHandles {
pub stats_leader: Arc<dyn LeaderElector>,
/// Single-owner lease for the chain-driven release sweep (issue #434).
pub release_sweep_leader: Arc<dyn LeaderElector>,
#[cfg(feature = "postgres")]
pub(crate) account_recovery: Option<Arc<crate::storage::postgres::recovery::RecoveryStore>>,
/// Account-scoped leases that fence Guardian executions (issue #254).
pub execution_leases: Arc<dyn ExecutionLeases>,
/// Shared publication store for the `/dashboard/stats` aggregate.
Expand Down Expand Up @@ -112,6 +114,8 @@ impl CoordinationHandles {
RELEASE_SWEEP_LEASE,
"single-process",
)),
#[cfg(feature = "postgres")]
account_recovery: None,
execution_leases: Arc::new(InMemoryExecutionLeases::new()),
stats_store: Arc::new(InMemoryStatsStore::new()),
block_watch_store: Arc::new(InMemoryBlockWatchStore::new()),
Expand Down Expand Up @@ -155,7 +159,14 @@ impl CoordinationHandles {
release_sweep_leader: Arc::new(PgLeaseElector::new(
pool.clone(),
RELEASE_SWEEP_LEASE,
holder_id,
holder_id.clone(),
)),
account_recovery: Some(Arc::new(
crate::storage::postgres::recovery::RecoveryStore::new(
pool.clone(),
cipher.clone(),
holder_id,
),
)),
execution_leases: Arc::new(PgExecutionLeases::new(pool.clone())),
stats_store: Arc::new(PgStatsStore::new(pool.clone(), cipher)),
Expand Down
Loading
Loading