Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import shutil
from pathlib import Path
from tempfile import mkdtemp

import pytest
from dbt_project import SCHEMA_NAME_SUFFIX, DbtProject

TEST_MODEL = "one"
MISSING_SCHEMA_SUFFIX = f"{SCHEMA_NAME_SUFFIX}_deferred_missing"


@pytest.mark.requires_dbt_version("1.5.0")
@pytest.mark.skip_for_dbt_fusion
@pytest.mark.skip_targets(["dremio"])
def test_artifacts_upload_skipped_when_deferred_elementary_schema_is_missing(
dbt_project: DbtProject,
):
# A `dbt run --defer --favor-state` against a target whose Elementary schema
# was never created (e.g. an ephemeral CI schema) must not fail at the
# on-run-end artifacts upload - the upload should be skipped, as it is
# when the relation is missing without deferral.
dbt_runner = dbt_project.dbt_runner

# Produce the manifest the deferred run uses as its state.
assert dbt_runner.run(select=TEST_MODEL)
state_dir = Path(mkdtemp(prefix="integration_tests_state_"))
shutil.copy(dbt_project.project_dir_path / "target" / "manifest.json", state_dir)

# The uploads are disabled by default in the tests for performance reasons.
deferred_run_vars = {
"disable_dbt_artifacts_autoupload": False,
"disable_run_results": False,
"disable_dbt_invocation_autoupload": False,
"schema_name_suffix": MISSING_SCHEMA_SUFFIX,
}
original_env_vars = dbt_runner.env_vars
dbt_runner.env_vars = {
**(original_env_vars or {}),
"DBT_DEFER": "1",
"DBT_FAVOR_STATE": "1",
"DBT_STATE": str(state_dir),
}
try:
deferred_run_success = dbt_runner.run(select=TEST_MODEL, vars=deferred_run_vars)
finally:
dbt_runner.env_vars = original_env_vars
shutil.rmtree(state_dir, ignore_errors=True)
dbt_runner.run_operation(
"elementary_tests.clear_env",
vars={"schema_name_suffix": MISSING_SCHEMA_SUFFIX},
)

assert (
deferred_run_success
), "Deferred run failed although the Elementary schema does not exist in the target."
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
{# data_monitoring_metrics_table is none except for integration-tests that test the get_anomaly_scores_query macro,
and in which case it holds mock history metrics #}
{%- set data_monitoring_metrics_table = elementary.get_elementary_relation(
"data_monitoring_metrics"
"data_monitoring_metrics", allow_deferred_fallback=true
) %}
{%- endif %}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,9 @@
) %}
{%- else %}
{%- set data_monitoring_metrics_relation = (
elementary.get_elementary_relation("data_monitoring_metrics")
elementary.get_elementary_relation(
"data_monitoring_metrics", allow_deferred_fallback=true
)
) %}
{%- endif %}
{%- set regular_bucket_times_query %}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
) %}
{% set timestamp_column = metric_properties.timestamp_column %}
{%- set data_monitoring_metrics_relation = elementary.get_elementary_relation(
"data_monitoring_metrics"
"data_monitoring_metrics", allow_deferred_fallback=true
) %}

with
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

{% macro default__get_latest_full_refresh(model_node) %}
{%- set dbt_run_results_relation = elementary.get_elementary_relation(
"dbt_run_results"
"dbt_run_results", allow_deferred_fallback=true
) %}
{% set query %}
select generated_at from {{ dbt_run_results_relation }}
Expand All @@ -19,7 +19,7 @@

{% macro fabric__get_latest_full_refresh(model_node) %}
{%- set dbt_run_results_relation = elementary.get_elementary_relation(
"dbt_run_results"
"dbt_run_results", allow_deferred_fallback=true
) %}
{% set query %}
select top 1 generated_at from {{ dbt_run_results_relation }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
) %}
{%- if not data_monitoring_metrics_table %}
{%- set data_monitoring_metrics_table = elementary.get_elementary_relation(
"data_monitoring_metrics"
"data_monitoring_metrics", allow_deferred_fallback=true
) %}
{%- endif %}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
full_table_name, temp_columns_snapshot_relation
) %}
{%- set schema_columns_snapshot_relation = elementary.get_elementary_relation(
"schema_columns_snapshot"
"schema_columns_snapshot", allow_deferred_fallback=true
) %}
{%- set previous_schema_time_query -%}
(select max(detected_at) from {{ schema_columns_snapshot_relation }} where lower(full_table_name) = lower('{{ full_table_name }}'))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
model_relation, full_table_name, into_relation=none
) %}
{%- set schema_columns_snapshot_relation = elementary.get_elementary_relation(
"schema_columns_snapshot"
"schema_columns_snapshot", allow_deferred_fallback=true
) %}
{%- set known_columns_query %}
select full_column_name from {{ schema_columns_snapshot_relation }}
Expand Down
4 changes: 2 additions & 2 deletions macros/edr/tests/on_run_end/handle_tests_results.sql
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@
{% if not test_tables_union_query %} {% do return(none) %} {% endif %}

{%- set target_relation = elementary.get_elementary_relation(
"data_monitoring_metrics"
"data_monitoring_metrics", allow_deferred_fallback=true
) -%}
{% if not target_relation %}
{% do elementary.raise_missing_elementary_models() %}
Expand Down Expand Up @@ -241,7 +241,7 @@
{% if not test_tables_union_query %} {% do return(none) %} {% endif %}

{%- set target_relation = elementary.get_elementary_relation(
"schema_columns_snapshot"
"schema_columns_snapshot", allow_deferred_fallback=true
) -%}
{% if not target_relation %}
{% do elementary.raise_missing_elementary_models() %}
Expand Down
2 changes: 1 addition & 1 deletion macros/edr/tests/test_execution_sla.sql
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@

{# Get reference to dbt_run_results table #}
{% set run_results_relation = elementary.get_elementary_relation(
"dbt_run_results"
"dbt_run_results", allow_deferred_fallback=true
) %}

{% if not run_results_relation %}
Expand Down
2 changes: 1 addition & 1 deletion macros/edr/tests/test_volume_threshold.sql
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@
) %}

{% set data_monitoring_metrics_table = elementary.get_elementary_relation(
"data_monitoring_metrics"
"data_monitoring_metrics", allow_deferred_fallback=true
) %}
{% set test_metrics_table = elementary.get_elementary_test_table(
elementary.get_elementary_test_table_name(), "metrics"
Expand Down
14 changes: 10 additions & 4 deletions macros/utils/graph/get_elementary_relation.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{% macro get_elementary_relation(identifier) %}
{% macro get_elementary_relation(identifier, allow_deferred_fallback=false) %}
{%- if execute %}
{%- set identifier_node = elementary.get_node(
"model.elementary." ~ identifier
Expand Down Expand Up @@ -27,9 +27,15 @@
{# Relation not found in the target schema. Under dbt deferral
(--favor-state / --defer) the Elementary models may exist only
in the deferred (e.g. prod) schema and not in the current
target. Construct a relation from the graph node coordinates
so the generated SQL references the correct schema instead of
rendering "from None". #}
target. Callers that render the relation into test SQL opt in
(allow_deferred_fallback=true) to a relation constructed from
the graph node coordinates, so the test SQL references the
correct schema instead of rendering "from None". Callers that
only check whether the relation exists (e.g. the on-run-end
artifact uploads) keep getting none, so a run against a target
whose Elementary schema was never created skips the upload
instead of failing. #}
{% if not allow_deferred_fallback %} {% do return(none) %} {% endif %}
{% set is_defer = (
(
invocation_args_dict.get("defer", false)
Expand Down
Loading