Skip to content

Commit 09ed1a6

Browse files
committed
Docs: Define the WAP materialization observation target
Signed-off-by: mday-io <mdaytn@gmail.com>
1 parent a60f3f8 commit 09ed1a6

2 files changed

Lines changed: 44 additions & 12 deletions

File tree

‎docs/reference/execution_observations.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,13 @@ SQLMesh calls `finish` for an audit before it runs later audits or completes
125125
write-audit-publish (WAP). A later audit or WAP publication can still fail. An audit
126126
query failure can produce a `finish` callback without an `AuditResult`. The
127127
[audits guide](../concepts/audits.md) describes audit behavior in plans and
128-
runs. The physical materialization callback ends after its surrounding
129-
transaction and session exit.
128+
runs.
129+
130+
For `physical/materialize`, `target` is the logical destination table from the
131+
snapshot. During WAP, SQLMesh writes to a staging destination, but the `target`
132+
fact remains the logical destination. The `finish` callback runs after the
133+
materialization's transaction and session exit. It does not report the result
134+
of later audits or WAP publication.
130135

131136
An observed operation can do no work, issue multiple statements, or trigger
132137
internal driver retries. The observation does not predict how many statements

‎tests/core/test_snapshot_evaluator.py‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5071,9 +5071,15 @@ def test_multiple_engine_virtual_layer(snapshot: Snapshot, adapters, make_snapsh
50715071
)
50725072

50735073

5074+
@pytest.mark.parametrize("observer_enabled", [False, True])
50745075
def test_wap_basic(
5075-
adapter_mock: Mock, make_snapshot: t.Callable[..., Snapshot], mocker: MockerFixture
5076+
adapter_mock: Mock,
5077+
make_snapshot: t.Callable[..., Snapshot],
5078+
mocker: MockerFixture,
5079+
observer_enabled: bool,
50765080
) -> None:
5081+
from sqlmesh.core.execution_observation import observer_scope
5082+
50775083
evaluator = SnapshotEvaluator(adapter_mock)
50785084

50795085
model = SqlModel(
@@ -5089,19 +5095,25 @@ def test_wap_basic(
50895095
expected_wap_table = "test_schema.test_table.branch_wap_12345678"
50905096
adapter_mock.wap_prepare.return_value = expected_wap_table
50915097

5092-
wap_id = evaluator.evaluate(
5093-
snapshot,
5094-
start="2020-01-01",
5095-
end="2020-01-01",
5096-
execution_time="2020-01-01",
5097-
snapshots={},
5098-
target_table_exists=True, # Use parameter to control table existence
5099-
)
5098+
observations = []
5099+
5100+
def observer(phase, facts, error):
5101+
observations.append((phase, dict(facts), error))
5102+
5103+
expected_table_name = snapshot.table_name()
5104+
with observer_scope(observer if observer_enabled else None):
5105+
wap_id = evaluator.evaluate(
5106+
snapshot,
5107+
start="2020-01-01",
5108+
end="2020-01-01",
5109+
execution_time="2020-01-01",
5110+
snapshots={},
5111+
target_table_exists=True, # Use parameter to control table existence
5112+
)
51005113

51015114
assert wap_id is not None
51025115
assert len(wap_id) == 8
51035116

5104-
expected_table_name = snapshot.table_name()
51055117
adapter_mock.wap_prepare.assert_called_once_with(expected_table_name, wap_id)
51065118
adapter_mock.replace_query.assert_called_once_with(
51075119
expected_wap_table,
@@ -5118,6 +5130,21 @@ def test_wap_basic(
51185130
source_columns=mocker.ANY,
51195131
)
51205132

5133+
materialization_observations = [
5134+
(phase, facts, error)
5135+
for phase, facts, error in observations
5136+
if facts.get("kind") == "physical" and facts.get("action") == "materialize"
5137+
]
5138+
if observer_enabled:
5139+
assert [
5140+
(phase, facts["target"], error) for phase, facts, error in materialization_observations
5141+
] == [
5142+
("start", expected_table_name, None),
5143+
("finish", expected_table_name, None),
5144+
]
5145+
else:
5146+
assert observations == []
5147+
51215148

51225149
def test_wap_model_wap_supported(
51235150
adapter_mock: Mock, make_snapshot: t.Callable[..., Snapshot], mocker: MockerFixture

0 commit comments

Comments
 (0)