Skip to content

Commit c7a1528

Browse files
committed
fix(snowflake): keep quoted and unquoted catalogs apart in last modified lookup
Group the tables by the catalog identifier instead of its name. In Snowflake an unquoted raw is the database RAW while a quoted "raw" is a different database, so grouping by name sent both to whichever information schema came first. Signed-off-by: Dhruv Gupta <todhruvg@gmail.com>
1 parent 516f0af commit c7a1528

2 files changed

Lines changed: 30 additions & 9 deletions

File tree

‎sqlmesh/core/engine_adapter/snowflake.py‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -758,21 +758,19 @@ def get_table_last_modified_ts(self, table_names: t.List[TableName]) -> t.List[i
758758
from sqlmesh.utils.date import to_timestamp
759759

760760
# INFORMATION_SCHEMA only contains the objects of the database it belongs to, so the tables
761-
# are grouped by database and each group is looked up in its own database's INFORMATION_SCHEMA
762-
catalogs_to_tables: t.Dict[str, t.List[exp.Table]] = {}
761+
# are grouped by database and each group is looked up in its own database's INFORMATION_SCHEMA.
762+
# The groups are keyed on the catalog identifier rather than its name, because quoting matters
763+
# in Snowflake: unquoted raw is the database RAW, while quoted "raw" is a different database
764+
catalogs_to_tables: t.Dict[t.Optional[exp.Identifier], t.List[exp.Table]] = {}
763765
for table_name in table_names:
764766
table = exp.to_table(table_name)
765-
catalogs_to_tables.setdefault(table.catalog, []).append(table)
767+
catalogs_to_tables.setdefault(table.args.get("catalog"), []).append(table)
766768

767769
results = []
768-
for tables in catalogs_to_tables.values():
770+
for catalog, tables in catalogs_to_tables.items():
769771
query = (
770772
exp.select("LAST_ALTERED")
771-
.from_(
772-
exp.table_(
773-
"TABLES", db="INFORMATION_SCHEMA", catalog=tables[0].args.get("catalog")
774-
)
775-
)
773+
.from_(exp.table_("TABLES", db="INFORMATION_SCHEMA", catalog=catalog))
776774
.where(
777775
exp.or_(
778776
*(

‎tests/core/engine_adapter/test_snowflake.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,29 @@ def test_session(
205205
assert to_sql_calls(adapter) == expected_exception_calls
206206

207207

208+
def test_get_table_last_modified_ts_quoted_and_unquoted_catalogs(
209+
snowflake_mocked_engine_adapter: SnowflakeEngineAdapter,
210+
) -> None:
211+
# unquoted raw resolves to the database RAW, quoted "raw" is a different database,
212+
# so the two must be looked up in two different information schemas
213+
adapter = snowflake_mocked_engine_adapter
214+
adapter.cursor.fetchall.side_effect = [
215+
[(datetime(2025, 1, 1, tzinfo=timezone.utc),)],
216+
[(datetime(2025, 1, 2, tzinfo=timezone.utc),)],
217+
]
218+
219+
result = adapter.get_table_last_modified_ts(['"raw".shopify.orders', "raw.shopify.customers"])
220+
221+
assert result == [
222+
to_timestamp(datetime(2025, 1, 1, tzinfo=timezone.utc)),
223+
to_timestamp(datetime(2025, 1, 2, tzinfo=timezone.utc)),
224+
]
225+
assert to_sql_calls(adapter) == [
226+
"""SELECT LAST_ALTERED FROM "raw".INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'orders' AND TABLE_SCHEMA = 'shopify'""",
227+
"""SELECT LAST_ALTERED FROM raw.INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'customers' AND TABLE_SCHEMA = 'shopify'""",
228+
]
229+
230+
208231
def test_comments(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture):
209232
adapter = make_mocked_engine_adapter(SnowflakeEngineAdapter)
210233

0 commit comments

Comments
 (0)