diff --git a/pkg-py/src/commons/_definitions/__init__.py b/pkg-py/src/commons/_definitions/__init__.py index 980d4fc6..bdeca3ad 100644 --- a/pkg-py/src/commons/_definitions/__init__.py +++ b/pkg-py/src/commons/_definitions/__init__.py @@ -12,7 +12,13 @@ authority, not this code. """ -from ._compile import attach_compiled_definitions, mixed_grain +from ._compile import ( + DefinitionAbsentColumnError, + DefinitionMixedGrainError, + DefinitionUnmatchedTableError, + attach_compiled_definitions, + mixed_grain, +) from ._registry import ( ExportRecord, Registry, @@ -26,6 +32,9 @@ ) __all__ = [ + "DefinitionAbsentColumnError", + "DefinitionMixedGrainError", + "DefinitionUnmatchedTableError", "ExportRecord", "Registry", "applied_text", diff --git a/pkg-py/src/commons/_definitions/_compile.py b/pkg-py/src/commons/_definitions/_compile.py index 4396fac1..04a196fa 100644 --- a/pkg-py/src/commons/_definitions/_compile.py +++ b/pkg-py/src/commons/_definitions/_compile.py @@ -18,7 +18,13 @@ from ._export import DefinitionExport, Ir from ._registry import ExportRecord -__all__ = ["attach_compiled_definitions", "mixed_grain"] +__all__ = [ + "DefinitionAbsentColumnError", + "DefinitionMixedGrainError", + "DefinitionUnmatchedTableError", + "attach_compiled_definitions", + "mixed_grain", +] _TARGETS = { "duckdb": "SQL(duckdb)", @@ -27,6 +33,18 @@ } +class DefinitionUnmatchedTableError(ValueError): + """An authored table declares definitions and matched no relation.""" + + +class DefinitionAbsentColumnError(ValueError): + """A definition names an authored column the matched relation lacks.""" + + +class DefinitionMixedGrainError(ValueError): + """A metric's dependency chain mixes row and aggregate grain.""" + + def mixed_grain(definitions: dict[str, DefinitionExport]) -> dict[str, bool]: """Which definitions mix row and aggregate grain, directly or inherited. @@ -126,59 +144,85 @@ def attach_compiled_definitions( Nothing is assigned until every table compiles, so a refusal leaves the dictionary untouched and the caller can attach it to another source. + + The refusals are checked in the order the definition-binding contract + pins, so a dictionary with several faults reports the same one in either + package: metric grain on every table, then the dialect, then each table + in authored order, its match before its columns. """ - exports: dict[str, dict[str, DefinitionExport]] = ( - getattr(dictionary, "definition_exports", None) or {} - ) - if bindings is not None: - _check_definitions_matched(exports, bindings) - compiled: dict[str, list[ExportRecord]] = {} - for table_name, entry in dictionary.tables.items(): - # A catalog import re-keys the dictionary to the warehouse's labels - # and records what the author called each table, which is how its - # exports are still found afterwards. - authored = getattr(entry, "authored_name", None) or table_name - definitions = exports.get(authored) or {} - if not definitions: - compiled[table_name] = [] - continue - if table_name not in exposed: - raise ValueError( - f"The data dictionary declares definitions on table " - f"{table_name!r}, which the data source does not expose. " - f"Exposed tables: {', '.join(sorted(exposed))}." - ) - target = _TARGETS.get(dialect) - if target is None: - raise ValueError( - f"Definitions on table {table_name!r} cannot be compiled for a " - f"{dialect!r} data source. commons lowers definitions to " - f"{', '.join(sorted(_TARGETS))}." - ) + exports: dict[str, dict[str, DefinitionExport]] = { + authored: definitions + for authored, definitions in ( + getattr(dictionary, "definition_exports", None) or {} + ).items() + if definitions + } + for authored, definitions in exports.items(): + _check_metric_grain(authored, definitions) + target = _TARGETS.get(dialect) + if exports and target is None: + raise ValueError( + f"Definitions on table {next(iter(exports))!r} cannot be compiled " + f"for a {dialect!r} data source. commons lowers definitions to " + f"{', '.join(sorted(_TARGETS))}." + ) + compiled: dict[str, list[ExportRecord]] = { + table_name: [] for table_name in dictionary.tables + } + for authored, definitions in exports.items(): + # A catalog import re-keys the dictionary to the warehouse's labels, + # so the bindings are how an authored table's exports find its entry. + table_name = _definition_table(authored, exposed, bindings) if bindings is not None: definitions = _bind_table( definitions, bindings["columns"].get(authored) or {}, table_name ) + assert target is not None compiled[table_name] = _compile_table(table_name, definitions, target) for table_name, entry in dictionary.tables.items(): entry.compiled_definitions = compiled[table_name] -def _check_definitions_matched( - exports: dict[str, dict[str, DefinitionExport]], bindings: dict[str, Any] -) -> None: - """Refuse a definition whose table the catalog selection left out. +def _definition_table( + authored: str, exposed: set[str], bindings: dict[str, Any] | None +) -> str: + """The table an authored table's definitions compile against. - Compiling it is impossible and dropping it is worse than failing: the - agent would be told about a governed definition that quietly is not there. + A table the catalog selection left out is refused rather than skipped: + the agent would otherwise be told about a governed definition that + quietly is not there. """ - for authored, definitions in exports.items(): - if definitions and bindings["tables"].get(authored) is None: + if bindings is None: + if authored not in exposed: raise ValueError( - f"Authored table {authored!r} declares definitions, and does " - f"not match an exposed relation. Name it as the data source " - f"selects it, or drop it from the data dictionary." + f"The data dictionary declares definitions on table " + f"{authored!r}, which the data source does not expose. " + f"Exposed tables: {', '.join(sorted(exposed))}." ) + return authored + label = bindings["tables"].get(authored) + if label is None: + raise DefinitionUnmatchedTableError( + f"Authored table {authored!r} declares definitions, and does " + f"not match an exposed relation. Name it as the data source " + f"selects it, or drop it from the data dictionary." + ) + return label + + +def _check_metric_grain(table: str, definitions: dict[str, DefinitionExport]) -> None: + grain = mixed_grain(definitions) + offenders = [ + name + for name, definition in definitions.items() + if grain[name] and definition.kind == "metric" + ] + if offenders: + raise DefinitionMixedGrainError( + f"Metric definition {offenders[0]!r} on table {table!r} cannot be " + f"compiled into one SQL expression. Its dependency chain mixes row " + f"and aggregate grain and would need a subquery rewrite." + ) def _bind_table( @@ -231,33 +275,19 @@ def _bind_selection( ) -> dict[str, Any] | None: """Rewrite a `COLUMNS(...)` selection to the discovered spellings. - A selection the author wrote out by name is bound strictly: naming a - column the warehouse does not have is a mistake worth reporting. A - selection derived from a pattern or from the whole table is not, because - the author named no column at all. It is resolved over the authored - columns, so a dictionary that still documents a dropped column would - otherwise take every wildcard definition on the table down with it, and - the refusal would name a column that appears nowhere in the expression. - The other side of that intersection is already tolerated: a warehouse - column the dictionary does not describe is simply not selected. + A selection from a pattern or from the whole table is resolved over the + authored columns, so it binds as strictly as one written out by name: a + dictionary that documents a column the relation lacks is refused. """ if selection is None: return None - named = selection.get("form") == "list" - bound = [] - for column in selection["columns"]: - path = _bind_path( - column["path"], columns, definition, table, required=named - ) - if path is not None: - bound.append({**column, "path": path}) - if not bound: - raise ValueError( - f"Definition {definition!r} on table {table!r} selects columns, " - f"and the selected relation has none of the ones the data " - f"dictionary describes." - ) - return {**selection, "columns": bound} + return { + **selection, + "columns": [ + {**column, "path": _bind_path(column["path"], columns, definition, table)} + for column in selection["columns"] + ], + } def _bind_path( @@ -265,19 +295,12 @@ def _bind_path( columns: dict[str, str | None], definition: str, table: str, - required: bool = True, ) -> Any: - """The physical spelling of the column a path leads with. - - Returns None for a column the relation does not have, when the caller - can drop it rather than refuse. - """ + """The physical spelling of the column a path leads with.""" segments = list(path) if isinstance(path, list) else [path] physical = columns.get(segments[0]) if physical is None: - if not required: - return None - raise ValueError( + raise DefinitionAbsentColumnError( f"Definition {definition!r} on table {table!r} references " f"authored column {segments[0]!r}, which is absent from the " f"selected relation." @@ -290,17 +313,6 @@ def _compile_table( table: str, definitions: dict[str, DefinitionExport], target: str ) -> list[ExportRecord]: grain = mixed_grain(definitions) - offenders = [ - name - for name, definition in definitions.items() - if grain[name] and definition.kind == "metric" - ] - if offenders: - raise ValueError( - f"Metric definition {offenders[0]!r} on table {table!r} cannot be " - f"compiled into one SQL expression. Its dependency chain mixes row " - f"and aggregate grain and would need a subquery rewrite." - ) markers = _reference_markers(definitions) # Only the source's own target is lowered, and ExportRecord carries that # one target. The R implementation emits every target up front and keeps diff --git a/pkg-py/tests/_warehouse.py b/pkg-py/tests/_warehouse.py index 90256c36..29094f8e 100644 --- a/pkg-py/tests/_warehouse.py +++ b/pkg-py/tests/_warehouse.py @@ -10,6 +10,8 @@ fake that answered both would hide a reader wired to the wrong warehouse. """ +from commons._catalog import MergedDictionary, Relation, merge_dictionary +from commons._data_dictionary import DataDictionary from commons._data_source import TableId from tests._shared import load_shared_fixture @@ -19,6 +21,7 @@ "column_row", "databricks_column_row", "databricks_relation_row", + "merge_shared_case", "relation_row", ] @@ -181,3 +184,26 @@ def list_tables(self) -> list[str]: def dialect(self) -> str: return "databricks" + + +def merge_shared_case(case: dict) -> MergedDictionary: + """Merge a shared fixture case's dictionary into its listed relations.""" + relations = {} + for entry in case["relations"]: + relations[entry["label"]] = Relation( + id=TableId( + catalog=entry.get("catalog"), + schema=entry.get("schema"), + table=entry["table"], + ), + kind=entry.get("kind"), + description=entry.get("description"), + ) + columns = {entry["label"]: entry["columns"] for entry in case["relations"]} + dictionary = DataDictionary.model_validate(case["dictionary"]) + return merge_dictionary( + dictionary, + relations, + describe_relation=lambda table_id: columns[table_id.label], + identifier_case=case["identifier_case"], + ) diff --git a/pkg-py/tests/test_catalog.py b/pkg-py/tests/test_catalog.py index 72e1253d..c5b393ff 100644 --- a/pkg-py/tests/test_catalog.py +++ b/pkg-py/tests/test_catalog.py @@ -18,7 +18,6 @@ CatalogAmbiguousTableError, CatalogClaimedRelationError, Manifest, - MergedDictionary, Relation, Selector, check_exclude, @@ -33,6 +32,7 @@ from commons._data_dictionary import DataDictionary from commons._data_source import TableId from tests._shared import load_shared_fixture +from tests._warehouse import merge_shared_case def relation(label: str, **kwargs) -> Relation: @@ -489,28 +489,6 @@ def test_exclusion_globs_match_the_shared_contract(): assert excluded(case["names"], case["patterns"]) == case["hidden"], case["name"] -def run_merge_case(case: dict) -> MergedDictionary: - relations = {} - for entry in case["relations"]: - relations[entry["label"]] = Relation( - id=TableId( - catalog=entry.get("catalog"), - schema=entry.get("schema"), - table=entry["table"], - ), - kind=entry.get("kind"), - description=entry.get("description"), - ) - columns = {entry["label"]: entry["columns"] for entry in case["relations"]} - dictionary = DataDictionary.model_validate(case["dictionary"]) - return merge_dictionary( - dictionary, - relations, - describe_relation=lambda table_id: columns[table_id.label], - identifier_case=case["identifier_case"], - ) - - _MERGE_ERRORS = { "ambiguous": CatalogAmbiguousTableError, "claimed": CatalogClaimedRelationError, @@ -525,10 +503,10 @@ def test_the_dictionary_merge_matches_the_shared_contract(): expect = case["expect"] if "error" in expect: with pytest.raises(_MERGE_ERRORS[expect["error"]]): - run_merge_case(case) + merge_shared_case(case) continue - merged = run_merge_case(case) + merged = merge_shared_case(case) assert list(merged.dictionary.tables) == list(expect["tables"]), case["name"] for label, want in expect["tables"].items(): table = merged.dictionary.tables[label] diff --git a/pkg-py/tests/test_definition_binding.py b/pkg-py/tests/test_definition_binding.py index 1dbccf2c..ad610ab0 100644 --- a/pkg-py/tests/test_definition_binding.py +++ b/pkg-py/tests/test_definition_binding.py @@ -8,81 +8,53 @@ import pytest -from commons._catalog._import import import_catalog from commons._data_dictionary import DataDictionary -from commons._definitions._compile import ( - _bind_ir, - _reference_markers, +from commons._definitions import ( + DefinitionAbsentColumnError, + DefinitionMixedGrainError, + DefinitionUnmatchedTableError, attach_compiled_definitions, ) -from commons._definitions._export import DefinitionExport, Ir -from tests._warehouse import FakeWarehouse +from commons._definitions._compile import _bind_ir +from commons._definitions._export import Ir +from tests._shared import load_shared_fixture +from tests._warehouse import merge_shared_case - -def dictionary(*definitions: dict, columns: list[dict] | None = None): - return DataDictionary.model_validate( - { - "tables": [ - { - "name": "sales", - "columns": columns - if columns is not None - else [{"name": "id", "type": "number(quantity)"}], - "definitions": list(definitions), - } - ] - } - ) +_BINDING_ERRORS = { + "absent_column": DefinitionAbsentColumnError, + "mixed_grain": DefinitionMixedGrainError, + "unmatched_table": DefinitionUnmatchedTableError, +} -def imported_sql(*definitions: dict, columns: list[dict] | None = None): - """Every definition's SQL, as a warehouse source would compile it.""" - imported = import_catalog( - FakeWarehouse(), dictionary=dictionary(*definitions, columns=columns) - ) +def compile_shared_case(case: dict) -> dict[str, dict[str, str]]: + merged = merge_shared_case(case) attach_compiled_definitions( - imported.dictionary, - "snowflake", - set(imported.tables), - imported.definition_bindings, - ) - assert imported.dictionary is not None - entry = imported.dictionary.tables["ANALYTICS.PUBLIC.SALES"] - return {record.name: record.sql for record in entry.compiled_definitions} - - -def test_a_column_is_lowered_with_the_spelling_the_warehouse_reported(): - compiled = imported_sql({"name": "total", "expr": "sum(id)"}) - - assert compiled["total"] == 'sum("ID")' - - -def test_a_sibling_reference_is_bound_through_its_own_expression(): - compiled = imported_sql( - {"name": "total", "expr": "sum(id)"}, - {"name": "doubled", "expr": "total * 2"}, - ) - - assert compiled["doubled"] == '(sum("ID")) * 2' - - -def test_a_selection_is_bound_column_by_column(): - compiled = imported_sql({"name": "any_set", "expr": "COLUMNS('^i') IS NOT NULL"}) - - assert '"ID"' in compiled["any_set"] - assert '"id"' not in compiled["any_set"] - - -def test_a_column_inside_a_case_branch_is_bound_too(): - # CASE keeps its branches as a list of condition-and-result pairs, so a - # walk that only descends into nodes it finds directly under a key walks - # past every column reference in them. - compiled = imported_sql( - {"name": "flagged", "expr": "CASE WHEN id > 0 THEN 1 ELSE 0 END"} + merged.dictionary, + case["backend"], + set(merged.dictionary.tables), + merged.definition_bindings, ) - - assert '"ID"' in compiled["flagged"] - assert '"id"' not in compiled["flagged"] + return { + label: {record.name: record.sql for record in table.compiled_definitions} + for label, table in merged.dictionary.tables.items() + if table.compiled_definitions + } + + +def test_definition_binding_matches_the_shared_contract(): + cases = load_shared_fixture("definition-binding")["cases"] + assert cases + for case in cases: + expect = case["expect"] + if "error" in expect: + try: + compile_shared_case(case) + except ValueError as error: + assert isinstance(error, _BINDING_ERRORS[expect["error"]]), case["name"] + continue + pytest.fail(f"{case['name']}: nothing was refused") + assert compile_shared_case(case) == expect["tables"], case["name"] def test_a_selection_is_bound_on_the_node_that_carries_it_too(): @@ -107,125 +79,20 @@ def test_a_selection_is_bound_on_the_node_that_carries_it_too(): assert [column["path"] for column in bound.attrs["selection"]["columns"]] == ["ID"] -def test_a_definition_over_a_column_the_relation_lacks_is_refused(): - # An authored column the warehouse never reported survives the merge, so - # the definition type-checks against the dictionary and only binding can - # catch that there is no such column to read. - with pytest.raises(ValueError, match="absent from the selected relation"): - imported_sql( - {"name": "total", "expr": "sum(absent)"}, - columns=[ - {"name": "id", "type": "number(quantity)"}, - {"name": "absent", "type": "number(quantity)"}, - ], - ) - - -def test_an_authored_table_that_matched_nothing_takes_its_definitions_with_it(): - imported = import_catalog( - FakeWarehouse(), - dictionary=DataDictionary.model_validate( - { - "tables": [ - { - "name": "unmatched", - "columns": [{"name": "id", "type": "number(quantity)"}], - "definitions": [{"name": "total", "expr": "sum(id)"}], - } - ] - } - ), - ) - - with pytest.raises(ValueError, match="does not match an exposed relation"): - attach_compiled_definitions( - imported.dictionary, - "snowflake", - set(imported.tables), - imported.definition_bindings, - ) - - def test_without_bindings_a_definition_keeps_the_authored_spelling(): - authored = dictionary({"name": "total", "expr": "sum(id)"}) - - attach_compiled_definitions(authored, "snowflake", {"sales"}) - - records = authored.tables["sales"].compiled_definitions - assert records[0].sql == 'sum("id")' - - -def test_a_bound_selection_cannot_be_mistaken_for_a_reference_marker(): - """A physical column named like a marker must not shadow a reference. - - Binding chooses the physical spelling, so nothing stops a warehouse - column from being named exactly like the marker a sibling reference is - substituted through. The selection carries that name outside the tree's - column nodes, which is the path the collision set used to miss. - """ - marker = "__commons_definition_reference_001__" - markers = _reference_markers( + authored = DataDictionary.model_validate( { - "base": DefinitionExport( - name="base", - label=None, - description=None, - details=None, - todo=None, - expression="sum(id)", - kind="metric", - type="number", - columns=["id"], - definitions=[], - selection={"columns": [{"path": [marker]}]}, - ) + "tables": [ + { + "name": "sales", + "columns": [{"name": "id", "type": "number(quantity)"}], + "definitions": [{"name": "total", "expr": "sum(id)"}], + } + ] } ) - assert markers["base"] != marker - - -def test_a_derived_selection_drops_a_column_the_relation_does_not_have(): - # A pattern names no column, so a dictionary that still documents one the - # warehouse dropped should not take every wildcard definition with it. - compiled = imported_sql( - {"name": "any_set", "expr": "COLUMNS('.*') IS NOT NULL"}, - columns=[ - {"name": "id", "type": "number(quantity)"}, - {"name": "ghost", "type": "number(quantity)"}, - ], - ) - - assert compiled["any_set"] == '"ID" IS NOT NULL' - - -def test_a_star_selection_drops_a_column_the_relation_does_not_have(): - compiled = imported_sql( - {"name": "any_set", "expr": "COLUMNS(*) IS NOT NULL"}, - columns=[ - {"name": "id", "type": "number(quantity)"}, - {"name": "ghost", "type": "number(quantity)"}, - ], - ) - - assert compiled["any_set"] == '"ID" IS NOT NULL' - - -def test_a_named_selection_still_refuses_a_column_the_relation_lacks(): - # The author wrote this column out, so its absence is worth reporting. - with pytest.raises(ValueError, match="authored column 'ghost'"): - imported_sql( - {"name": "any_set", "expr": "COLUMNS([id, ghost]) IS NOT NULL"}, - columns=[ - {"name": "id", "type": "number(quantity)"}, - {"name": "ghost", "type": "number(quantity)"}, - ], - ) - + attach_compiled_definitions(authored, "snowflake", {"sales"}) -def test_a_derived_selection_left_with_nothing_is_refused(): - with pytest.raises(ValueError, match="has none of the ones"): - imported_sql( - {"name": "any_set", "expr": "COLUMNS('.*') IS NOT NULL"}, - columns=[{"name": "ghost", "type": "number(quantity)"}], - ) + records = authored.tables["sales"].compiled_definitions + assert records[0].sql == 'sum("id")' diff --git a/pkg-r/NEWS.md b/pkg-r/NEWS.md index e8c91a03..4ab55757 100644 --- a/pkg-r/NEWS.md +++ b/pkg-r/NEWS.md @@ -10,6 +10,8 @@ * When a data dictionary cannot be matched to the warehouse catalog, the error now has a condition class you can catch: `commons_catalog_ambiguous_table_error` when an authored table name matches more than one selected relation, `commons_catalog_claimed_relation_error` when two authored tables match the same relation, and `commons_catalog_ambiguous_column_error` when an authored column name matches more than one discovered column. +* When a definition in a data dictionary cannot be bound to the warehouse catalog, the error now has a condition class you can catch: `commons_definition_mixed_grain_error` when a metric mixes row and aggregate grain, `commons_definition_unmatched_table_error` when the definition's table matches no selected relation, and `commons_definition_absent_column_error` when the definition names a column the relation does not have. + * The CSS classes on `run_r`'s display are now `commons-run-display`, `commons-run-details`, `commons-run-code`, and `commons-run-plot`, without the `-r`, because the Python package's code tool uses the same classes. App CSS that targets the old `commons-run-r-*` names needs updating. # commons 0.1.1 diff --git a/pkg-r/R/catalog.R b/pkg-r/R/catalog.R index 1ee5948a..6dd94833 100644 --- a/pkg-r/R/catalog.R +++ b/pkg-r/R/catalog.R @@ -382,8 +382,7 @@ catalog_merge_dictionary <- function( relations = relations, definition_bindings = list( tables = matches, - columns = column_matches, - strict = TRUE + columns = column_matches ) ) } diff --git a/pkg-r/R/definition-compile.R b/pkg-r/R/definition-compile.R index 8513282d..72564959 100644 --- a/pkg-r/R/definition-compile.R +++ b/pkg-r/R/definition-compile.R @@ -102,6 +102,7 @@ definition_check_metric_grain <- function(tables, call) { "Metric definition {.val {names(definitions)[invalid]}} on table {.val {table$name}} cannot be compiled into one SQL expression.", i = "The dependency chain mixes row and aggregate grain and requires a subquery rewrite." ), + class = "commons_definition_mixed_grain_error", call = call ) } @@ -133,7 +134,6 @@ definition_bind_one <- function( bound_ir <- definition_bind_ir( ir, binding$columns, - binding$strict, definition$name, table, call @@ -141,7 +141,6 @@ definition_bind_one <- function( bound_selection <- definition_bind_selection( selection, binding$columns, - binding$strict, definition$name, table, call @@ -272,26 +271,22 @@ definition_source_binding <- function(source, table, call) { call = call ) } - return(list(table = table, columns = NULL, strict = FALSE)) + return(list(table = table, columns = NULL)) } selected <- bindings$tables[[table]] if (is.null(selected) || is.na(selected)) { cli::cli_abort( "Authored table {.val {table}} with definitions does not match an exposed relation.", + class = "commons_definition_unmatched_table_error", call = call ) } - list( - table = selected, - columns = bindings$columns[[table]], - strict = isTRUE(bindings$strict) - ) + list(table = selected, columns = bindings$columns[[table]]) } definition_bind_ir <- function( node, columns, - strict, definition, table, call @@ -306,7 +301,6 @@ definition_bind_ir <- function( node$path <- definition_bind_column_path( node$path, columns, - strict, definition, table, call @@ -316,7 +310,6 @@ definition_bind_ir <- function( node, definition_bind_ir, columns = columns, - strict = strict, definition = definition, table = table, call = call @@ -326,7 +319,6 @@ definition_bind_ir <- function( definition_bind_selection <- function( selection, columns, - strict, definition, table, call @@ -338,7 +330,6 @@ definition_bind_selection <- function( column$path <- definition_bind_column_path( column$path, columns, - strict, definition, table, call @@ -351,7 +342,6 @@ definition_bind_selection <- function( definition_bind_column_path <- function( path, columns, - strict, definition, table, call @@ -361,13 +351,11 @@ definition_bind_column_path <- function( } physical <- columns[[path[[1]]]] if (is.null(physical) || is.na(physical)) { - if (strict) { - cli::cli_abort( - "Definition {.val {definition}} on table {.val {table}} references authored column {.val {path[[1]]}}, which is absent from the selected relation.", - call = call - ) - } - return(path) + cli::cli_abort( + "Definition {.val {definition}} on table {.val {table}} references authored column {.val {path[[1]]}}, which is absent from the selected relation.", + class = "commons_definition_absent_column_error", + call = call + ) } path[[1]] <- physical path diff --git a/pkg-r/tests/testthat/fixtures/shared/definition-binding.json b/pkg-r/tests/testthat/fixtures/shared/definition-binding.json new file mode 100644 index 00000000..2034dcd8 --- /dev/null +++ b/pkg-r/tests/testthat/fixtures/shared/definition-binding.json @@ -0,0 +1,462 @@ +{ + "cases": [ + { + "name": "a column is lowered with the spelling Snowflake reported", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [ + { "name": "total", "expr": "sum(id)" }, + { "name": "doubled", "expr": "total * 2" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { + "total": "sum(\"ID\")", + "doubled": "(sum(\"ID\")) * 2" + } + } + } + }, + { + "name": "a column is lowered with the spelling Databricks reported", + "backend": "databricks", + "identifier_case": "lower", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [{ "name": "Amount", "type": "number(quantity)" }], + "definitions": [{ "name": "total", "expr": "sum(Amount)" }] + } + ] + }, + "relations": [ + { + "label": "main.default.sales", + "catalog": "main", + "schema": "default", + "table": "sales", + "columns": [{ "column": "amount", "type": "bigint" }] + } + ], + "expect": { + "tables": { + "main.default.sales": { "total": "sum(`amount`)" } + } + } + }, + { + "name": "a column inside a CASE branch is bound", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [ + { "name": "flagged", "expr": "CASE WHEN id > 0 THEN 1 ELSE 0 END" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { + "flagged": "CASE WHEN \"ID\" > 0 THEN 1 ELSE 0 END" + } + } + } + }, + { + "name": "a pattern selection is bound column by column", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [ + { "name": "any_set", "expr": "COLUMNS('^i') IS NOT NULL" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { "any_set": "\"ID\" IS NOT NULL" } + } + } + }, + { + "name": "a column the relation lacks is refused", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [{ "name": "total", "expr": "sum(ghost)" }] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "a named selection refuses a column the relation lacks", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [ + { "name": "any_set", "expr": "COLUMNS([id, ghost]) IS NOT NULL" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "a pattern selection refuses a column the relation lacks", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [ + { "name": "any_set", "expr": "COLUMNS('.*') IS NOT NULL" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "a star selection refuses a column the relation lacks", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [ + { "name": "any_set", "expr": "COLUMNS(*) IS NOT NULL" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "a table with definitions that matched no relation is refused", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "unmatched", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [{ "name": "total", "expr": "sum(id)" }] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "unmatched_table" } + }, + { + "name": "a physical column named like a reference marker does not shadow the reference", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { + "name": "__COMMONS_DEFINITION_REFERENCE_001__", + "type": "number(quantity)" + } + ], + "definitions": [ + { "name": "base", "expr": "id > 0" }, + { + "name": "flagged", + "expr": "base AND __COMMONS_DEFINITION_REFERENCE_001__ > 0" + } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [ + { "column": "ID", "type": "NUMBER(38,0)" }, + { + "column": "__commons_definition_reference_001__", + "type": "NUMBER(38,0)" + } + ] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { + "base": "\"ID\" > 0", + "flagged": "(\"ID\" > 0) AND \"__commons_definition_reference_001__\" > 0" + } + } + } + }, + { + "name": "a marker-named column reachable only through a selection does not shadow the reference", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { + "name": "__COMMONS_DEFINITION_REFERENCE_001__", + "type": "number(quantity)" + } + ], + "definitions": [ + { "name": "base", "expr": "id > 0" }, + { + "name": "flagged", + "expr": "base AND COLUMNS('^__') IS NOT NULL" + } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [ + { "column": "ID", "type": "NUMBER(38,0)" }, + { + "column": "__commons_definition_reference_001__", + "type": "NUMBER(38,0)" + } + ] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { + "base": "\"ID\" > 0", + "flagged": "(\"ID\" > 0) AND \"__commons_definition_reference_001__\" IS NOT NULL" + } + } + } + }, + { + "name": "a mixed-grain metric is refused before a column the relation lacks", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [ + { "name": "flag", "expr": "id > avg(id)" }, + { + "name": "flagged", + "expr": "sum(case when flag then 1 else 0 end)" + }, + { "name": "total", "expr": "sum(ghost)" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "mixed_grain" } + }, + { + "name": "tables are checked in authored order, a column before a later table's match", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [{ "name": "total", "expr": "sum(ghost)" }] + }, + { + "name": "unmatched", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [{ "name": "total", "expr": "sum(id)" }] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "tables are checked in authored order, a match before a later table's column", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "unmatched", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [{ "name": "total", "expr": "sum(id)" }] + }, + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [{ "name": "total", "expr": "sum(ghost)" }] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "unmatched_table" } + } + ] +} diff --git a/pkg-r/tests/testthat/helper-catalog-rows.R b/pkg-r/tests/testthat/helper-catalog-rows.R index 531fe61a..7286034c 100644 --- a/pkg-r/tests/testthat/helper-catalog-rows.R +++ b/pkg-r/tests/testthat/helper-catalog-rows.R @@ -179,3 +179,40 @@ catalog_labels_reply <- function(case) { ) } } + +# Merge a shared fixture case's dictionary into its listed relations, as +# catalog-merge.json and definition-binding.json both describe them. +merge_shared_case <- function(case) { + relations <- list() + columns <- list() + for (entry in case$relations) { + id_args <- Filter(Negate(is.null), entry[c("catalog", "schema", "table")]) + relations[[entry$label]] <- list( + id = do.call(DBI::Id, id_args), + kind = entry$kind, + description = entry$description + ) + columns[[entry$label]] <- entry$columns + } + describe <- function(con, id, ...) { + rows <- columns[[table_id_label(id)]] + data.frame( + column = vapply(rows, `[[`, character(1), "column"), + type = vapply(rows, function(row) row$type %||% NA_character_, character(1)), + nullable = vapply(rows, function(row) row$nullable %||% NA, logical(1)), + description = vapply( + rows, + function(row) row$description %||% NA_character_, + character(1) + ), + stringsAsFactors = FALSE + ) + } + catalog_merge_dictionary( + new_data_dictionary(case$dictionary), + relations, + NULL, + describe, + case$identifier_case + ) +} diff --git a/pkg-r/tests/testthat/test-catalog-merge.R b/pkg-r/tests/testthat/test-catalog-merge.R index 710a3bf6..6f15c978 100644 --- a/pkg-r/tests/testthat/test-catalog-merge.R +++ b/pkg-r/tests/testthat/test-catalog-merge.R @@ -40,41 +40,7 @@ test_that("the dictionary merge matches the shared contract", { cases <- shared_fixture("catalog-merge")$merge expect_gt(length(cases), 0L) for (case in cases) { - dictionary <- new_data_dictionary(case$dictionary) - relations <- list() - columns <- list() - for (entry in case$relations) { - id_args <- Filter(Negate(is.null), entry[c("catalog", "schema", "table")]) - relations[[entry$label]] <- list( - id = do.call(DBI::Id, id_args), - kind = entry$kind, - description = entry$description - ) - columns[[entry$label]] <- entry$columns - } - describe <- function(con, id, ...) { - rows <- columns[[table_id_label(id)]] - data.frame( - column = vapply(rows, `[[`, character(1), "column"), - type = vapply(rows, function(row) row$type %||% NA_character_, character(1)), - nullable = vapply(rows, function(row) row$nullable %||% NA, logical(1)), - description = vapply( - rows, - function(row) row$description %||% NA_character_, - character(1) - ), - stringsAsFactors = FALSE - ) - } - run <- function() { - catalog_merge_dictionary( - dictionary, - relations, - NULL, - describe, - case$identifier_case - ) - } + run <- function() merge_shared_case(case) if (!is.null(case$expect$error)) { expect_error(run(), class = merge_error_class[[case$expect$error]]) next diff --git a/pkg-r/tests/testthat/test-data-dictionary.R b/pkg-r/tests/testthat/test-data-dictionary.R index 682e9fd3..70cfb57a 100644 --- a/pkg-r/tests/testthat/test-data-dictionary.R +++ b/pkg-r/tests/testthat/test-data-dictionary.R @@ -242,7 +242,6 @@ test_that("warehouse metadata supplements an authored dictionary", { merged$definition_bindings$columns$orders, c(amount = "AMOUNT", missing = NA_character_) ) - expect_true(merged$definition_bindings$strict) expect_length(merged$dictionary$relationships, 1L) entry <- dictionary_entry_text( merged$dictionary, diff --git a/pkg-r/tests/testthat/test-definition-binding.R b/pkg-r/tests/testthat/test-definition-binding.R new file mode 100644 index 00000000..0bfcbc0d --- /dev/null +++ b/pkg-r/tests/testthat/test-definition-binding.R @@ -0,0 +1,48 @@ +# The definition-binding contract in tests/shared/, run against the R +# implementation. The Python suite runs the same cases from the same file. + +binding_error_class <- c( + absent_column = "commons_definition_absent_column_error", + mixed_grain = "commons_definition_mixed_grain_error", + unmatched_table = "commons_definition_unmatched_table_error" +) + +compile_binding_case <- function(case) { + local_mocked_bindings( + is_snowflake_connection = function(con) identical(case$backend, "snowflake"), + is_databricks_connection = function(con) identical(case$backend, "databricks") + ) + merged <- merge_shared_case(case) + source <- new_data_source( + structure(list(), class = "mock_connection"), + tables = names(merged$dictionary$tables), + owned = FALSE, + definition_bindings = merged$definition_bindings + ) + compiled <- definition_compile_source(case$dictionary, source) + lapply(compiled$tables, function(table) { + sql <- lapply(table$definitions, `[[`, "sql") + names(sql) <- vapply(table$definitions, `[[`, character(1), "name") + sql + }) +} + +test_that("definition binding matches the shared contract", { + cases <- shared_fixture("definition-binding")$cases + expect_gt(length(cases), 0L) + for (case in cases) { + if (!is.null(case$expect$error)) { + expect_error( + compile_binding_case(case), + class = binding_error_class[[case$expect$error]], + info = case$name + ) + next + } + expect_identical( + compile_binding_case(case), + case$expect$tables, + label = case$name + ) + } +}) diff --git a/pkg-r/tests/testthat/test-definition-compile.R b/pkg-r/tests/testthat/test-definition-compile.R index fe5d0f80..6bbd63fb 100644 --- a/pkg-r/tests/testthat/test-definition-compile.R +++ b/pkg-r/tests/testthat/test-definition-compile.R @@ -277,8 +277,7 @@ test_that("source selection binds authored warehouse identifiers", { order_total = "ORDER_TOTAL", tile_size = "TILE_SIZE" ) - ), - strict = TRUE + ) ) source <- definition_mock_source("ANALYTICS.PUBLIC.ORDERS", bindings) compiled <- definition_compile_fixture("core.yaml", source) @@ -301,32 +300,6 @@ test_that("source selection binds authored warehouse identifiers", { ) }) -test_that("missing warehouse bindings fail before SQL can run", { - local_mocked_bindings( - is_snowflake_connection = function(con) TRUE, - is_databricks_connection = function(con) FALSE - ) - bindings <- list( - tables = c(orders = "ANALYTICS.PUBLIC.ORDERS"), - columns = list( - orders = c( - status_cd = "STATUS_CD", - order_total = "ORDER_TOTAL", - tile_size = NA_character_ - ) - ), - strict = TRUE - ) - - expect_error( - definition_compile_fixture( - "core.yaml", - definition_mock_source("ANALYTICS.PUBLIC.ORDERS", bindings) - ), - "authored column.*tile_size" - ) -}) - test_that("the source backend selects the destination target", { local_mocked_bindings( is_snowflake_connection = function(con) FALSE, @@ -421,8 +394,7 @@ test_that("physical columns cannot be confused with sibling definitions", { ) bindings <- list( tables = c(values = "ANALYTICS.PUBLIC.VALUES"), - columns = list(values = c(amount = "AMOUNT")), - strict = TRUE + columns = list(values = c(amount = "AMOUNT")) ) compiled <- definition_compile_source( raw, diff --git a/tests/shared/README.md b/tests/shared/README.md index d4a493da..9eaedf80 100644 --- a/tests/shared/README.md +++ b/tests/shared/README.md @@ -43,6 +43,7 @@ The Python suite reads this directory directly. The R suite cannot. `testthat` n - **Table descriptions.** `describe-table.json` pins the SQL-native DuckDB column types shown by `describe_table` and confirms that inspecting the schema does not expose row values. Both suites create and inspect the same real DuckDB table. - **Catalog rows.** `catalog-rows.json` pins how a warehouse listing becomes relations and columns: which rows are relations at all, what kind each is, which comments count as prose, and where a `DESCRIBE` reply stops being columns. The rows are what Snowflake's `SHOW OBJECTS` and `DESC TABLE`, and Databricks' `system.information_schema.tables` and `DESCRIBE TABLE`, actually return. Running those queries is each language's own business; agreeing on their replies is not. Hand-maintained, since no binary generates it, and booleans travel as strings so the file reads the same from both JSON readers. - **The catalog merge.** `catalog-merge.json` pins the contract between a warehouse listing and an authored dictionary: the three limits (object cap, prompt threshold, search probe bound), exclusion-glob behavior over bare table names, and merge scenarios — authored prose wins, warehouse types and nullability win, identifier case folds per backend, ambiguity is an error — each with the expected merged tables and definition bindings. Error cases pin a slug rather than message text, because the wording belongs to each language. +- **Definition binding.** `definition-binding.json` pins how a governed definition is bound to the relation a warehouse catalog matched: every column it names, including those inside a `CASE` branch or a `COLUMNS(...)` selection, is lowered with the spelling the warehouse reported, and a physical column named like a sibling reference's marker cannot take that reference's place. A selection from a pattern or from `*` binds as strictly as one written out by name, so a dictionary that documents a column the relation lacks is refused whichever form names it. A dictionary with several faults is refused for the same one in either package: a mixed-grain metric on any table first, then each table in authored order, its match before its columns. Each case reuses `catalog-merge.json`'s dictionary and relation shapes and runs the real merge first. Refusals pin a slug rather than message text, because the wording belongs to each language. Hand-maintained. - **Catalog access errors.** `catalog-access-errors.json` pins how a failed access probe is read: a SQLSTATE and a message become `authorization`, `transient`, or `unknown`, which decides what the user is told and whether the answer is cached per relation. A case carrying a `sql` field asserts that only the driver's own complaint is read: each suite attaches that statement the way its own drivers repeat it back, and the expected kind stays the one the message alone earns, so a relation cannot classify itself by its name. Hand-maintained, since the cases are the failures real drivers report rather than anything a binary emits. An absent SQLSTATE travels as an empty string, because JSON has no way to spell R's `NA_character_`. - **Session identity and access precedence.** `catalog-session-changed.json` pins which parts of a warehouse session a refusal reports as changed, so neither package tells a Databricks user that a role moved on a backend that has none. `catalog-access-precedence.json` pins what construction reports when several named relations fail at once: every relation the listing reported is probed before anything is raised, and a name the warehouse never listed is reported ahead of a refusal. Both are hand-maintained, and neither carries message text, since the wording belongs to each language: the session cases travel as snapshot field names, the precedence cases as relation labels and outcome slugs. - **Relation labels.** `catalog-relation-labels.json` pins the label a selection entry ends up under, which is what the agent sees in a table listing and what every access error names. A confirmed entry is qualified with the connection's namespace only when it named no schema of its own; an entry the warehouse never reported keeps the authored name whole; a relation the warehouse reports without a namespace of its own keeps its bare name, since the label is also the id every later query names it by. Each case names the backend it runs against, because only a Databricks `SHOW TABLES` reply can report a relation with no namespace. The warehouse's own id is kept separately as the relation's identity, since it carries that backend's casing. Hand-maintained. Identifiers are already case-folded, so these cases pin qualification alone and leave case folding to `catalog-merge.json`. diff --git a/tests/shared/definition-binding.json b/tests/shared/definition-binding.json new file mode 100644 index 00000000..2034dcd8 --- /dev/null +++ b/tests/shared/definition-binding.json @@ -0,0 +1,462 @@ +{ + "cases": [ + { + "name": "a column is lowered with the spelling Snowflake reported", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [ + { "name": "total", "expr": "sum(id)" }, + { "name": "doubled", "expr": "total * 2" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { + "total": "sum(\"ID\")", + "doubled": "(sum(\"ID\")) * 2" + } + } + } + }, + { + "name": "a column is lowered with the spelling Databricks reported", + "backend": "databricks", + "identifier_case": "lower", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [{ "name": "Amount", "type": "number(quantity)" }], + "definitions": [{ "name": "total", "expr": "sum(Amount)" }] + } + ] + }, + "relations": [ + { + "label": "main.default.sales", + "catalog": "main", + "schema": "default", + "table": "sales", + "columns": [{ "column": "amount", "type": "bigint" }] + } + ], + "expect": { + "tables": { + "main.default.sales": { "total": "sum(`amount`)" } + } + } + }, + { + "name": "a column inside a CASE branch is bound", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [ + { "name": "flagged", "expr": "CASE WHEN id > 0 THEN 1 ELSE 0 END" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { + "flagged": "CASE WHEN \"ID\" > 0 THEN 1 ELSE 0 END" + } + } + } + }, + { + "name": "a pattern selection is bound column by column", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [ + { "name": "any_set", "expr": "COLUMNS('^i') IS NOT NULL" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { "any_set": "\"ID\" IS NOT NULL" } + } + } + }, + { + "name": "a column the relation lacks is refused", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [{ "name": "total", "expr": "sum(ghost)" }] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "a named selection refuses a column the relation lacks", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [ + { "name": "any_set", "expr": "COLUMNS([id, ghost]) IS NOT NULL" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "a pattern selection refuses a column the relation lacks", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [ + { "name": "any_set", "expr": "COLUMNS('.*') IS NOT NULL" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "a star selection refuses a column the relation lacks", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [ + { "name": "any_set", "expr": "COLUMNS(*) IS NOT NULL" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "a table with definitions that matched no relation is refused", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "unmatched", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [{ "name": "total", "expr": "sum(id)" }] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "unmatched_table" } + }, + { + "name": "a physical column named like a reference marker does not shadow the reference", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { + "name": "__COMMONS_DEFINITION_REFERENCE_001__", + "type": "number(quantity)" + } + ], + "definitions": [ + { "name": "base", "expr": "id > 0" }, + { + "name": "flagged", + "expr": "base AND __COMMONS_DEFINITION_REFERENCE_001__ > 0" + } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [ + { "column": "ID", "type": "NUMBER(38,0)" }, + { + "column": "__commons_definition_reference_001__", + "type": "NUMBER(38,0)" + } + ] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { + "base": "\"ID\" > 0", + "flagged": "(\"ID\" > 0) AND \"__commons_definition_reference_001__\" > 0" + } + } + } + }, + { + "name": "a marker-named column reachable only through a selection does not shadow the reference", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { + "name": "__COMMONS_DEFINITION_REFERENCE_001__", + "type": "number(quantity)" + } + ], + "definitions": [ + { "name": "base", "expr": "id > 0" }, + { + "name": "flagged", + "expr": "base AND COLUMNS('^__') IS NOT NULL" + } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [ + { "column": "ID", "type": "NUMBER(38,0)" }, + { + "column": "__commons_definition_reference_001__", + "type": "NUMBER(38,0)" + } + ] + } + ], + "expect": { + "tables": { + "ANALYTICS.PUBLIC.SALES": { + "base": "\"ID\" > 0", + "flagged": "(\"ID\" > 0) AND \"__commons_definition_reference_001__\" IS NOT NULL" + } + } + } + }, + { + "name": "a mixed-grain metric is refused before a column the relation lacks", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [ + { "name": "flag", "expr": "id > avg(id)" }, + { + "name": "flagged", + "expr": "sum(case when flag then 1 else 0 end)" + }, + { "name": "total", "expr": "sum(ghost)" } + ] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "mixed_grain" } + }, + { + "name": "tables are checked in authored order, a column before a later table's match", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [{ "name": "total", "expr": "sum(ghost)" }] + }, + { + "name": "unmatched", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [{ "name": "total", "expr": "sum(id)" }] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "absent_column" } + }, + { + "name": "tables are checked in authored order, a match before a later table's column", + "backend": "snowflake", + "identifier_case": "upper", + "dictionary": { + "tables": [ + { + "name": "unmatched", + "columns": [{ "name": "id", "type": "number(quantity)" }], + "definitions": [{ "name": "total", "expr": "sum(id)" }] + }, + { + "name": "sales", + "columns": [ + { "name": "id", "type": "number(quantity)" }, + { "name": "ghost", "type": "number(quantity)" } + ], + "definitions": [{ "name": "total", "expr": "sum(ghost)" }] + } + ] + }, + "relations": [ + { + "label": "ANALYTICS.PUBLIC.SALES", + "catalog": "ANALYTICS", + "schema": "PUBLIC", + "table": "SALES", + "columns": [{ "column": "ID", "type": "NUMBER(38,0)" }] + } + ], + "expect": { "error": "unmatched_table" } + } + ] +}