diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index 1f2ddc5e4ed..f0a5399db85 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -458,6 +458,30 @@ where Ok(self) } + /// Only a started block stream registers the cancel guard that unassigning removes. + fn can_observe_unassign(&self) -> bool { + self.cancel_handle.is_some() + } + + async fn back_off_before_restart( + &mut self, + err: Error, + reason: RestartReason, + ) -> Result, SubgraphRunnerError> { + if self.is_canceled() { + return self.cancel(); + } + self.metrics.subgraph.deployment_status.failed(); + error!(self.logger, "Subgraph failed to restart, retrying: {:#}", err; + "attempt" => self.state.backoff.attempt, + "retry_delay_s" => self.state.backoff.delay().as_secs()); + self.state.backoff.sleep_async().await; + if self.is_canceled() { + return self.cancel(); + } + Ok(RunnerState::Restarting { reason }) + } + pub async fn run(self) -> Result<(), SubgraphRunnerError> { self.run_inner(false).await.map(|_| ()) } @@ -499,7 +523,12 @@ where reason: StopReason::Canceled, } } else { - self.restart(reason).await? + match self.restart(reason).await { + Err(SubgraphRunnerError::Unknown(e)) if self.can_observe_unassign() => { + self.back_off_before_restart(e, reason).await? + } + res => res?, + } } } @@ -563,7 +592,7 @@ where if block.trigger_count() == 0 && self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD && !self.inputs.store.is_deployment_synced() - && !close_to_chain_head(&block_ptr, &self.inputs.chain.chain_head_ptr().await?, 1000) + && !self.is_close_to_chain_head(&block_ptr).await && !max_end_block_reached { // Skip this block and continue with the same stream @@ -604,6 +633,18 @@ where } } + /// An unreadable chain head counts as close, so the block is processed rather than skipped. + async fn is_close_to_chain_head(&self, block_ptr: &BlockPtr) -> bool { + match self.inputs.chain.chain_head_ptr().await { + Ok(chain_head_ptr) => close_to_chain_head(block_ptr, &chain_head_ptr, 1000), + Err(e) => { + warn!(self.logger, "Failed to read the chain head, processing the block"; + "error" => format!("{:#}", e)); + true + } + } + } + /// Handle a revert event and determine the next state. /// /// This is the state machine wrapper around `handle_revert` that handles @@ -1295,11 +1336,19 @@ where if self.state.should_try_unfail_non_deterministic { // If the deployment head advanced, we can unfail // the non-deterministic error (if there's any). - let outcome = self + let outcome = match self .inputs .store .unfail_non_deterministic_error(&block_ptr) - .await?; + .await + { + Ok(outcome) => outcome, + Err(e) => { + warn!(self.logger, "Failed to unfail subgraph, trying again on the next block"; + "error" => format!("{:#}", e)); + UnfailOutcome::BehindErrorBlock + } + }; // Stop trying unless we're still behind the error block. if outcome != UnfailOutcome::BehindErrorBlock { @@ -1381,7 +1430,19 @@ where let message = format!("{:#}", e).replace('\n', "\t"); let err = anyhow!("{}, code: {}", message, LogCode::SubgraphSyncingFailure); - self.fail_subgraph(message, Some(block_ptr), true).await?; + // The block's PoI is already written, so a restart would resume past it. + while let Err(e) = self + .fail_subgraph(message.clone(), Some(block_ptr.clone()), true) + .await + { + if self.is_canceled() { + return Ok(Action::Stop); + } + error!(self.logger, "Failed to record deterministic error, retrying: {:#}", e; + "attempt" => self.state.backoff.attempt, + "retry_delay_s" => self.state.backoff.delay().as_secs()); + self.state.backoff.sleep_async().await; + } Err(err) } @@ -1408,8 +1469,19 @@ where // // If we don't do this check we would keep adding the same error to the // database. - if !self.inputs.store.health().await?.is_failed() { - self.fail_subgraph(message, Some(block_ptr), false).await?; + let already_failed = match self.inputs.store.health().await { + Ok(health) => health.is_failed(), + Err(e) => { + warn!(self.logger, "Failed to read subgraph health"; + "error" => format!("{:#}", e)); + false + } + }; + if !already_failed + && let Err(e) = self.fail_subgraph(message, Some(block_ptr), false).await + { + warn!(self.logger, "Failed to record non-deterministic error"; + "error" => format!("{:#}", e)); } // Sleep before restarting. diff --git a/tests/tests/writer_restart.rs b/tests/tests/writer_restart.rs index c96e15aeae5..3f3a3e0146e 100644 --- a/tests/tests/writer_restart.rs +++ b/tests/tests/writer_restart.rs @@ -1,12 +1,14 @@ +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use graph::env::EnvVars; use graph::prelude::diesel::connection::SimpleConnection; use graph::prelude::diesel::dsl::sql; -use graph::prelude::diesel::sql_types::Text; +use graph::prelude::diesel::sql_types::{BigInt, Integer, Text}; use graph::prelude::diesel::{self, Connection, PgConnection, RunQueryDsl}; -use graph::prelude::{BlockNumber, SubgraphStore as _}; +use graph::prelude::{BlockNumber, BlockPtr, SubgraphStore as _}; use graph_tests::fixture::ethereum::{chain, empty_block, genesis, push_test_log}; use graph_tests::fixture::{self, TestContext, test_ptr}; use graph_tests::recipe::RunnerTestRecipe; @@ -34,15 +36,21 @@ fn db() -> PgConnection { PgConnection::establish(&url).expect("connect to the test database") } -/// Fails the first transaction that moves the deployment head to `block` or -/// later; the sequence is not rolled back with it, so later writes succeed. -fn fail_first_write_reaching(ctx: &TestContext, block: BlockNumber) { +fn deployment_namespace(ctx: &TestContext) -> String { let id = ctx.deployment.id; - let nsp: String = diesel::select(sql::(&format!( + diesel::select(sql::(&format!( "(select name from deployment_schemas where id = {id})" ))) .get_result(&mut db()) - .unwrap(); + .unwrap() +} + +/// Fails the first transaction that moves the deployment head to `block` or +/// later; the sequence is not rolled back with it, so later writes succeed. +/// Returns the deployment's namespace. +fn fail_first_write_reaching(ctx: &TestContext, block: BlockNumber) -> String { + let id = ctx.deployment.id; + let nsp = deployment_namespace(ctx); db().batch_execute(&format!( " create sequence {nsp}.failed_writes; @@ -65,6 +73,7 @@ fn fail_first_write_reaching(ctx: &TestContext, block: BlockNumber) { " )) .unwrap(); + nsp } #[graph::test] @@ -160,3 +169,319 @@ async fn data_source_restart_keeps_healthy_writer() { )); assert!(writer_before.restart().await.unwrap().is_none()); } + +/// From the write that `fail_first_write_reaching` fails until `end`, every +/// statement that reads `subgraphs.` fails, as it would with Postgres +/// down; `table` keeps its rows behind a view while the outage lasts. +struct Outage { + nsp: String, + table: &'static str, + ended: bool, +} + +impl Outage { + fn after_injected_failure(nsp: String, table: &'static str) -> Self { + db().batch_execute(&format!( + " + create sequence {nsp}.outage_reads; + + create function {nsp}.database_reachable() returns boolean + language plpgsql volatile as $$ + begin + if (select is_called from {nsp}.failed_writes) then + perform nextval('{nsp}.outage_reads'); + raise exception 'injected outage: database unavailable'; + end if; + return true; + end; + $$; + + alter table subgraphs.{table} rename to {table}_behind_outage; + create view subgraphs.{table} as + select * from subgraphs.{table}_behind_outage + where {nsp}.database_reachable(); + " + )) + .unwrap(); + Outage { + nsp, + table, + ended: false, + } + } + + fn failed_reads(&self) -> i64 { + let nsp = &self.nsp; + diesel::select(sql::(&format!( + "(select case when is_called then last_value else 0 end from {nsp}.outage_reads)" + ))) + .get_result(&mut db()) + .unwrap() + } + + fn head_behind_outage(&self, ctx: &TestContext) -> BlockNumber { + let id = ctx.deployment.id; + diesel::select(sql::(&format!( + "(select block_number from subgraphs.head_behind_outage where id = {id})" + ))) + .get_result(&mut db()) + .unwrap() + } + + fn end(&mut self) { + if !self.ended { + let table = self.table; + db().batch_execute(&format!( + " + drop view subgraphs.{table}; + alter table subgraphs.{table}_behind_outage rename to {table}; + " + )) + .unwrap(); + self.ended = true; + } + } +} + +impl Drop for Outage { + fn drop(&mut self) { + self.end(); + } +} + +/// Waits until the injected failures happened `failures` times, while the +/// runner is still running. +async fn wait_for_failures(count: impl Fn() -> i64, failures: i64, run: &mut RunningRunner) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while count() < failures { + tokio::select! { + result = &mut *run => panic!("the runner ended during the outage: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(20)) => {} + } + assert!( + tokio::time::Instant::now() < deadline, + "failed {} times, waited for {failures}", + count() + ); + } +} + +/// Fails every attempt to record a fatal error until `end`; the block writes +/// that carry the error succeed. +struct ErrorRecordingOutage { + nsp: String, + ended: bool, +} + +impl ErrorRecordingOutage { + fn start(ctx: &TestContext) -> Self { + let id = ctx.deployment.id; + let nsp = deployment_namespace(ctx); + db().batch_execute(&format!( + " + create sequence {nsp}.failed_records; + + create function {nsp}.fail_error_record() returns trigger + language plpgsql as $$ + begin + perform nextval('{nsp}.failed_records'); + raise exception 'injected outage: database unavailable'; + end; + $$; + + create trigger fail_error_record_{nsp} + before update on subgraphs.deployment + for each row + when (new.id = {id} and new.fatal_error is not null) + execute function {nsp}.fail_error_record(); + " + )) + .unwrap(); + ErrorRecordingOutage { nsp, ended: false } + } + + fn failed_records(&self) -> i64 { + let nsp = &self.nsp; + diesel::select(sql::(&format!( + "(select case when is_called then last_value else 0 end from {nsp}.failed_records)" + ))) + .get_result(&mut db()) + .unwrap() + } + + fn end(&mut self) { + if !self.ended { + let nsp = &self.nsp; + db().batch_execute(&format!( + "drop trigger fail_error_record_{nsp} on subgraphs.deployment;" + )) + .unwrap(); + self.ended = true; + } + } +} + +impl Drop for ErrorRecordingOutage { + fn drop(&mut self) { + self.end(); + } +} + +fn head_number(ctx: &TestContext) -> BlockNumber { + let id = ctx.deployment.id; + diesel::select(sql::(&format!( + "(select block_number from subgraphs.head where id = {id})" + ))) + .get_result(&mut db()) + .unwrap() +} + +type RunningRunner = Pin>>>; + +/// The runner's future is not `Send`, so the test polls it instead of spawning it. +fn start_runner( + runner: graph_core::subgraph::SubgraphRunner< + graph_chain_ethereum::Chain, + graph_runtime_wasm::RuntimeHostBuilder, + >, +) -> RunningRunner { + Box::pin(async move { runner.run_for_test(false).await.map(|_| ()) }) +} + +async fn outage_test_context(name: &str) -> (TestContext, BlockPtr) { + configure_writer_for_injected_failures(); + + let RunnerTestRecipe { stores, test_info } = RunnerTestRecipe::new(name, "end-block").await; + + let mut blocks = vec![genesis()]; + for number in 1..=5 { + let mut block = empty_block(blocks.last().unwrap().ptr(), test_ptr(number)); + push_test_log(&mut block, number.to_string()); + blocks.push(block); + } + + let chain = chain(&test_info.test_name, blocks, &stores, None).await; + let mut env_vars = EnvVars::default(); + env_vars.subgraph_error_retry_ceil = Duration::from_secs(1); + let ctx = fixture::setup(&test_info, &stores, &chain, None, Some(env_vars)).await; + (ctx, test_ptr(5)) +} + +#[graph::test] +async fn store_errors_while_handling_a_non_deterministic_error_keep_the_runner() { + let _one_test = ONE_TEST_AT_A_TIME.lock().await; + let (ctx, stop_block) = outage_test_context("store_errors_while_handling_keep_runner").await; + + let nsp = fail_first_write_reaching(&ctx, 2); + let mut outage = Outage::after_injected_failure(nsp, "deployment"); + + let mut run = start_runner(ctx.runner(stop_block.clone()).await); + // `health()` and `fail_subgraph()` in the non-deterministic error arm. + wait_for_failures(|| outage.failed_reads(), 2, &mut run).await; + outage.end(); + + tokio::time::timeout(Duration::from_secs(60), run) + .await + .expect("the runner did not finish after the outage") + .unwrap(); + + let head = ctx + .store + .least_block_ptr(&ctx.deployment.hash) + .await + .unwrap(); + assert_eq!(head, Some(stop_block)); + let status = ctx.indexing_status().await; + assert!(status.fatal_error.is_none()); +} + +#[graph::test] +async fn store_errors_at_restart_back_off_and_retry() { + let _one_test = ONE_TEST_AT_A_TIME.lock().await; + let (ctx, stop_block) = outage_test_context("store_errors_at_restart_retry").await; + + let nsp = fail_first_write_reaching(&ctx, 2); + let mut outage = Outage::after_injected_failure(nsp, "head"); + + let mut run = start_runner(ctx.runner(stop_block.clone()).await); + // Two `store.restart()` calls, so the first one was retried. + wait_for_failures(|| outage.failed_reads(), 2, &mut run).await; + assert_eq!(outage.head_behind_outage(&ctx), 1); + outage.end(); + + tokio::time::timeout(Duration::from_secs(60), run) + .await + .expect("the runner did not finish after the outage") + .unwrap(); + + let head = ctx + .store + .least_block_ptr(&ctx.deployment.hash) + .await + .unwrap(); + assert_eq!(head, Some(stop_block)); + let status = ctx.indexing_status().await; + assert!(status.fatal_error.is_none()); +} + +#[graph::test] +async fn unassigned_runner_stops_while_restart_backs_off() { + let _one_test = ONE_TEST_AT_A_TIME.lock().await; + let (ctx, stop_block) = outage_test_context("unassigned_runner_stops_in_backoff").await; + + let nsp = fail_first_write_reaching(&ctx, 2); + let outage = Outage::after_injected_failure(nsp, "head"); + + let runner = ctx.runner(stop_block).await; + let instances = runner.context().instances.clone(); + let mut run = start_runner(runner); + wait_for_failures(|| outage.failed_reads(), 2, &mut run).await; + + instances.remove(&ctx.deployment.id); + + // One backoff interval is at most `subgraph_error_retry_ceil` (1s) plus its jitter. + let result = tokio::time::timeout(Duration::from_secs(3), run) + .await + .expect("the unassigned runner kept retrying"); + if let Err(e) = result { + assert!(!format!("{e:#}").contains("injected outage"), "{e:#}"); + } +} + +#[graph::test] +async fn deterministic_error_is_recorded_without_passing_its_block() { + let _one_test = ONE_TEST_AT_A_TIME.lock().await; + configure_writer_for_injected_failures(); + + let RunnerTestRecipe { stores, test_info } = + RunnerTestRecipe::new("deterministic_error_recorded_in_place", "fatal-error").await; + + let mut blocks = vec![genesis()]; + for number in 1..=5 { + blocks.push(empty_block(blocks.last().unwrap().ptr(), test_ptr(number))); + } + + let chain = chain(&test_info.test_name, blocks, &stores, None).await; + let mut env_vars = EnvVars::default(); + env_vars.subgraph_error_retry_ceil = Duration::from_secs(1); + let ctx = fixture::setup(&test_info, &stores, &chain, None, Some(env_vars)).await; + + let mut outage = ErrorRecordingOutage::start(&ctx); + + let mut run = start_runner(ctx.runner(test_ptr(5)).await); + // The fatal-error subgraph fails deterministically at block 3. + wait_for_failures(|| outage.failed_records(), 2, &mut run).await; + assert!(head_number(&ctx) <= 3); + outage.end(); + + let result = tokio::time::timeout(Duration::from_secs(60), run) + .await + .expect("the runner did not finish after the outage"); + assert!(result.is_err()); + + assert!(head_number(&ctx) <= 3); + let status = ctx.indexing_status().await; + let err = status.fatal_error.unwrap(); + assert_eq!(err.block.number, 3.into()); + assert!(err.deterministic); +}