From f5175b9aa284888d5bcb5f1f71f5cd9dd2dfbbe0 Mon Sep 17 00:00:00 2001 From: Dhruv Gupta Date: Wed, 7 Oct 2026 09:54:23 -0400 Subject: [PATCH] fix: skip deferred relation fallback for artifact upload callers Since #1006, get_elementary_relation builds a relation from the graph node coordinates when the physical relation is missing and --defer/--favor-state is active, so deferred anomaly tests do not render "from None". The same fallback also fired for the on-run-end artifact uploads during `dbt run --defer --favor-state`, where the Elementary schema of the current target is never created, turning the previous graceful skip into a "schema does not exist" failure. Make the fallback opt-in through an allow_deferred_fallback parameter. Only the callers that render the relation into test SQL pass it; the artifact upload callers keep getting none when the relation does not exist and skip the upload as before. Fixes #1036 --- .../test_deferred_run_artifacts_upload.py | 55 +++++++++++++++++++ .../get_anomaly_scores_query.sql | 2 +- .../get_buckets_configuration.sql | 4 +- .../dimension_monitoring_query.sql | 2 +- .../get_latest_full_refresh.sql | 4 +- .../monitors_query/metric_stability_query.sql | 2 +- .../get_columns_changes_query.sql | 2 +- .../get_columns_snapshot_query.sql | 2 +- .../tests/on_run_end/handle_tests_results.sql | 4 +- macros/edr/tests/test_execution_sla.sql | 2 +- macros/edr/tests/test_volume_threshold.sql | 2 +- .../utils/graph/get_elementary_relation.sql | 14 +++-- 12 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 integration_tests/tests/test_dbt_artifacts/test_deferred_run_artifacts_upload.py diff --git a/integration_tests/tests/test_dbt_artifacts/test_deferred_run_artifacts_upload.py b/integration_tests/tests/test_dbt_artifacts/test_deferred_run_artifacts_upload.py new file mode 100644 index 000000000..693a31550 --- /dev/null +++ b/integration_tests/tests/test_dbt_artifacts/test_deferred_run_artifacts_upload.py @@ -0,0 +1,55 @@ +import shutil +from pathlib import Path +from tempfile import mkdtemp + +import pytest +from dbt_project import SCHEMA_NAME_SUFFIX, DbtProject + +TEST_MODEL = "one" +MISSING_SCHEMA_SUFFIX = f"{SCHEMA_NAME_SUFFIX}_deferred_missing" + + +@pytest.mark.requires_dbt_version("1.5.0") +@pytest.mark.skip_for_dbt_fusion +@pytest.mark.skip_targets(["dremio"]) +def test_artifacts_upload_skipped_when_deferred_elementary_schema_is_missing( + dbt_project: DbtProject, +): + # A `dbt run --defer --favor-state` against a target whose Elementary schema + # was never created (e.g. an ephemeral CI schema) must not fail at the + # on-run-end artifacts upload - the upload should be skipped, as it is + # when the relation is missing without deferral. + dbt_runner = dbt_project.dbt_runner + + # Produce the manifest the deferred run uses as its state. + assert dbt_runner.run(select=TEST_MODEL) + state_dir = Path(mkdtemp(prefix="integration_tests_state_")) + shutil.copy(dbt_project.project_dir_path / "target" / "manifest.json", state_dir) + + # The uploads are disabled by default in the tests for performance reasons. + deferred_run_vars = { + "disable_dbt_artifacts_autoupload": False, + "disable_run_results": False, + "disable_dbt_invocation_autoupload": False, + "schema_name_suffix": MISSING_SCHEMA_SUFFIX, + } + original_env_vars = dbt_runner.env_vars + dbt_runner.env_vars = { + **(original_env_vars or {}), + "DBT_DEFER": "1", + "DBT_FAVOR_STATE": "1", + "DBT_STATE": str(state_dir), + } + try: + deferred_run_success = dbt_runner.run(select=TEST_MODEL, vars=deferred_run_vars) + finally: + dbt_runner.env_vars = original_env_vars + shutil.rmtree(state_dir, ignore_errors=True) + dbt_runner.run_operation( + "elementary_tests.clear_env", + vars={"schema_name_suffix": MISSING_SCHEMA_SUFFIX}, + ) + + assert ( + deferred_run_success + ), "Deferred run failed although the Elementary schema does not exist in the target." diff --git a/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql b/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql index 3e76e3dab..c6ccbb3da 100644 --- a/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql +++ b/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql @@ -16,7 +16,7 @@ {# data_monitoring_metrics_table is none except for integration-tests that test the get_anomaly_scores_query macro, and in which case it holds mock history metrics #} {%- set data_monitoring_metrics_table = elementary.get_elementary_relation( - "data_monitoring_metrics" + "data_monitoring_metrics", allow_deferred_fallback=true ) %} {%- endif %} diff --git a/macros/edr/data_monitoring/data_monitors_configuration/get_buckets_configuration.sql b/macros/edr/data_monitoring/data_monitors_configuration/get_buckets_configuration.sql index cbbba17d1..6f5925772 100644 --- a/macros/edr/data_monitoring/data_monitors_configuration/get_buckets_configuration.sql +++ b/macros/edr/data_monitoring/data_monitors_configuration/get_buckets_configuration.sql @@ -74,7 +74,9 @@ ) %} {%- else %} {%- set data_monitoring_metrics_relation = ( - elementary.get_elementary_relation("data_monitoring_metrics") + elementary.get_elementary_relation( + "data_monitoring_metrics", allow_deferred_fallback=true + ) ) %} {%- endif %} {%- set regular_bucket_times_query %} diff --git a/macros/edr/data_monitoring/monitors_query/dimension_monitoring_query.sql b/macros/edr/data_monitoring/monitors_query/dimension_monitoring_query.sql index ceb80546a..5fe0eba5c 100644 --- a/macros/edr/data_monitoring/monitors_query/dimension_monitoring_query.sql +++ b/macros/edr/data_monitoring/monitors_query/dimension_monitoring_query.sql @@ -25,7 +25,7 @@ ) %} {% set timestamp_column = metric_properties.timestamp_column %} {%- set data_monitoring_metrics_relation = elementary.get_elementary_relation( - "data_monitoring_metrics" + "data_monitoring_metrics", allow_deferred_fallback=true ) %} with diff --git a/macros/edr/data_monitoring/monitors_query/get_latest_full_refresh.sql b/macros/edr/data_monitoring/monitors_query/get_latest_full_refresh.sql index 08f998c61..5a6b4c10b 100644 --- a/macros/edr/data_monitoring/monitors_query/get_latest_full_refresh.sql +++ b/macros/edr/data_monitoring/monitors_query/get_latest_full_refresh.sql @@ -4,7 +4,7 @@ {% macro default__get_latest_full_refresh(model_node) %} {%- set dbt_run_results_relation = elementary.get_elementary_relation( - "dbt_run_results" + "dbt_run_results", allow_deferred_fallback=true ) %} {% set query %} select generated_at from {{ dbt_run_results_relation }} @@ -19,7 +19,7 @@ {% macro fabric__get_latest_full_refresh(model_node) %} {%- set dbt_run_results_relation = elementary.get_elementary_relation( - "dbt_run_results" + "dbt_run_results", allow_deferred_fallback=true ) %} {% set query %} select top 1 generated_at from {{ dbt_run_results_relation }} diff --git a/macros/edr/data_monitoring/monitors_query/metric_stability_query.sql b/macros/edr/data_monitoring/monitors_query/metric_stability_query.sql index 6b31c6049..3b9f6f647 100644 --- a/macros/edr/data_monitoring/monitors_query/metric_stability_query.sql +++ b/macros/edr/data_monitoring/monitors_query/metric_stability_query.sql @@ -16,7 +16,7 @@ ) %} {%- if not data_monitoring_metrics_table %} {%- set data_monitoring_metrics_table = elementary.get_elementary_relation( - "data_monitoring_metrics" + "data_monitoring_metrics", allow_deferred_fallback=true ) %} {%- endif %} diff --git a/macros/edr/data_monitoring/schema_changes/get_columns_changes_query.sql b/macros/edr/data_monitoring/schema_changes/get_columns_changes_query.sql index cd31efa1e..eecb2fdc7 100644 --- a/macros/edr/data_monitoring/schema_changes/get_columns_changes_query.sql +++ b/macros/edr/data_monitoring/schema_changes/get_columns_changes_query.sql @@ -2,7 +2,7 @@ full_table_name, temp_columns_snapshot_relation ) %} {%- set schema_columns_snapshot_relation = elementary.get_elementary_relation( - "schema_columns_snapshot" + "schema_columns_snapshot", allow_deferred_fallback=true ) %} {%- set previous_schema_time_query -%} (select max(detected_at) from {{ schema_columns_snapshot_relation }} where lower(full_table_name) = lower('{{ full_table_name }}')) diff --git a/macros/edr/data_monitoring/schema_changes/get_columns_snapshot_query.sql b/macros/edr/data_monitoring/schema_changes/get_columns_snapshot_query.sql index 410fdfac9..5cb085ff3 100644 --- a/macros/edr/data_monitoring/schema_changes/get_columns_snapshot_query.sql +++ b/macros/edr/data_monitoring/schema_changes/get_columns_snapshot_query.sql @@ -2,7 +2,7 @@ model_relation, full_table_name, into_relation=none ) %} {%- set schema_columns_snapshot_relation = elementary.get_elementary_relation( - "schema_columns_snapshot" + "schema_columns_snapshot", allow_deferred_fallback=true ) %} {%- set known_columns_query %} select full_column_name from {{ schema_columns_snapshot_relation }} diff --git a/macros/edr/tests/on_run_end/handle_tests_results.sql b/macros/edr/tests/on_run_end/handle_tests_results.sql index b41114080..f360d16f0 100644 --- a/macros/edr/tests/on_run_end/handle_tests_results.sql +++ b/macros/edr/tests/on_run_end/handle_tests_results.sql @@ -171,7 +171,7 @@ {% if not test_tables_union_query %} {% do return(none) %} {% endif %} {%- set target_relation = elementary.get_elementary_relation( - "data_monitoring_metrics" + "data_monitoring_metrics", allow_deferred_fallback=true ) -%} {% if not target_relation %} {% do elementary.raise_missing_elementary_models() %} @@ -241,7 +241,7 @@ {% if not test_tables_union_query %} {% do return(none) %} {% endif %} {%- set target_relation = elementary.get_elementary_relation( - "schema_columns_snapshot" + "schema_columns_snapshot", allow_deferred_fallback=true ) -%} {% if not target_relation %} {% do elementary.raise_missing_elementary_models() %} diff --git a/macros/edr/tests/test_execution_sla.sql b/macros/edr/tests/test_execution_sla.sql index aaac4240e..4d1fb6324 100644 --- a/macros/edr/tests/test_execution_sla.sql +++ b/macros/edr/tests/test_execution_sla.sql @@ -190,7 +190,7 @@ {# Get reference to dbt_run_results table #} {% set run_results_relation = elementary.get_elementary_relation( - "dbt_run_results" + "dbt_run_results", allow_deferred_fallback=true ) %} {% if not run_results_relation %} diff --git a/macros/edr/tests/test_volume_threshold.sql b/macros/edr/tests/test_volume_threshold.sql index e681830f8..544637b5e 100644 --- a/macros/edr/tests/test_volume_threshold.sql +++ b/macros/edr/tests/test_volume_threshold.sql @@ -159,7 +159,7 @@ ) %} {% set data_monitoring_metrics_table = elementary.get_elementary_relation( - "data_monitoring_metrics" + "data_monitoring_metrics", allow_deferred_fallback=true ) %} {% set test_metrics_table = elementary.get_elementary_test_table( elementary.get_elementary_test_table_name(), "metrics" diff --git a/macros/utils/graph/get_elementary_relation.sql b/macros/utils/graph/get_elementary_relation.sql index 2ddf7d2f5..7d5dddb40 100644 --- a/macros/utils/graph/get_elementary_relation.sql +++ b/macros/utils/graph/get_elementary_relation.sql @@ -1,4 +1,4 @@ -{% macro get_elementary_relation(identifier) %} +{% macro get_elementary_relation(identifier, allow_deferred_fallback=false) %} {%- if execute %} {%- set identifier_node = elementary.get_node( "model.elementary." ~ identifier @@ -27,9 +27,15 @@ {# Relation not found in the target schema. Under dbt deferral (--favor-state / --defer) the Elementary models may exist only in the deferred (e.g. prod) schema and not in the current - target. Construct a relation from the graph node coordinates - so the generated SQL references the correct schema instead of - rendering "from None". #} + target. Callers that render the relation into test SQL opt in + (allow_deferred_fallback=true) to a relation constructed from + the graph node coordinates, so the test SQL references the + correct schema instead of rendering "from None". Callers that + only check whether the relation exists (e.g. the on-run-end + artifact uploads) keep getting none, so a run against a target + whose Elementary schema was never created skips the upload + instead of failing. #} + {% if not allow_deferred_fallback %} {% do return(none) %} {% endif %} {% set is_defer = ( ( invocation_args_dict.get("defer", false)