From fa6c9e3c757893fa6034dda8241dc2cd1ca302e7 Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Wed, 9 Sep 2026 13:36:01 -0600 Subject: [PATCH 1/7] test(py): a live Snowflake and Databricks harness, configured by environment Every warehouse test in pkg-py drives a fake replaying canned rows, so nothing had ever run a statement against a real warehouse. This adds the other half. Configuration is environment variables in both languages, with the names held in tests/shared/live-warehouse-env.json so the R harness (which still reads R options, and moves next) cannot drift from this one. Connection variables are the vendor's own, which is what the Python clients and odbc::databricks() already read, so one export block serves both languages and the vendor CLIs. Without a token the Databricks connector runs the OAuth browser flow, which is what makes an SSO-only account usable. The fixture tables are the three the corpus dictionaries in definition-export/valid/ describe, so definition-warehouse-sql.json can be executed rather than only agreed on: each definition runs on the warehouse and on the same rows in DuckDB, and the answers must match unless the record carries a note, which is where an emitter documents a deliberate divergence. Reading the exemption off the notes means a divergence cannot be claimed without also being documented. Rows live in the shared fixture, not in the provisioning SQL, because both engines must hold the same data for the comparison to mean anything. One definition is recorded as unrunnable: data-dict's own DuckDB translation of interval(amount + 1, days) will not bind over a double, and amount has to be fractional for `precise threshold` to compare anything. test_live_warehouse_fixture.py covers what needs no warehouse, so CI checks the rows, the loader and the generator rather than leaving them unexercised until someone with credentials runs them. Refs kata whhh. --- pkg-py/pyproject.toml | 8 +- pkg-py/tests/README.md | 51 +++ pkg-py/tests/_live_warehouse.py | 244 +++++++++++ pkg-py/tests/test_live_warehouse_fixture.py | 137 ++++++ pkg-py/tests/test_live_warehouses.py | 239 +++++++++++ pkg-py/uv.lock | 331 ++++++++++++++- .../fixtures/shared/live-warehouse-env.json | 69 ++++ .../fixtures/shared/live-warehouse-rows.json | 389 ++++++++++++++++++ scripts/provision-live-warehouse.py | 166 ++++++++ tests/shared/README.md | 2 + tests/shared/live-warehouse-env.json | 69 ++++ tests/shared/live-warehouse-rows.json | 389 ++++++++++++++++++ 12 files changed, 2084 insertions(+), 10 deletions(-) create mode 100644 pkg-py/tests/README.md create mode 100644 pkg-py/tests/_live_warehouse.py create mode 100644 pkg-py/tests/test_live_warehouse_fixture.py create mode 100644 pkg-py/tests/test_live_warehouses.py create mode 100644 pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json create mode 100644 pkg-r/tests/testthat/fixtures/shared/live-warehouse-rows.json create mode 100755 scripts/provision-live-warehouse.py create mode 100644 tests/shared/live-warehouse-env.json create mode 100644 tests/shared/live-warehouse-rows.json diff --git a/pkg-py/pyproject.toml b/pkg-py/pyproject.toml index 1969340f7..bbd6047ba 100644 --- a/pkg-py/pyproject.toml +++ b/pkg-py/pyproject.toml @@ -43,10 +43,16 @@ shiny = [ ] [dependency-groups] -dev = ["pytest>=8", "pytest-asyncio", "ruff", "pyrefly", "pandas>=2", "polars>=1", "pins>=0.9", "commons[shiny]", "matplotlib>=3.8", "great-tables>=1.0.0", "plotnine>=0.15.8"] +# pytz is what DuckDB reaches for to hand back a TIMESTAMPTZ, which the live +# warehouse fixture rows use so their offsets survive the round trip. +dev = ["pytest>=8", "pytest-asyncio", "ruff", "pyrefly", "pandas>=2", "polars>=1", "pins>=0.9", "commons[shiny]", "matplotlib>=3.8", "great-tables>=1.0.0", "plotnine>=0.15.8", "pytz"] docs = [ "great-docs>=0.17", ] +# Drivers for the live warehouse tests, which skip without a configured +# warehouse. Kept out of `dev` so an ordinary contributor does not install a +# warehouse client to run the suite. +live-warehouse = ["databricks-sqlalchemy>=2", "snowflake-sqlalchemy>=1.6"] [build-system] requires = ["hatchling"] diff --git a/pkg-py/tests/README.md b/pkg-py/tests/README.md new file mode 100644 index 000000000..ba79ef647 --- /dev/null +++ b/pkg-py/tests/README.md @@ -0,0 +1,51 @@ +# Running the tests + +`uv run pytest` runs everything. Nothing here needs credentials: a test that +cannot reach what it needs skips. + +## Live warehouse tests + +`test_live_warehouses.py` runs against a real Snowflake or Databricks and +skips without one. It exists because every other warehouse test drives a fake +replaying canned rows, which cannot tell us what a warehouse actually returns. + +Install the drivers, which are deliberately outside the `dev` group so an +ordinary contributor does not install a warehouse client: + +``` +uv sync --extra live-warehouse +``` + +Point at a warehouse. The connection variables are the vendor's own, so one +export block also serves the Databricks CLI and the R suite: + +``` +export DATABRICKS_HOST=https://example.cloud.databricks.com +export DATABRICKS_HTTP_PATH=/sql/1.0/warehouses/abc123 +export DATABRICKS_TOKEN=dapi... # omit for the OAuth browser flow +export COMMONS_TEST_DATABRICKS_SCHEMA=main.commons_live +``` + +With `DATABRICKS_TOKEN` unset the connector runs the OAuth user-to-machine +flow, which opens a browser and reuses an SSO session, so an SSO-only account +needs no long-lived secret. `tests/shared/live-warehouse-env.json` is the full +list of variables, including the optional ones that gate individual tests. + +Create the tables the tests expect: + +``` +python3 ../scripts/provision-live-warehouse.py databricks main.commons_live +``` + +That prints SQL to run; it does not connect. The rows come from +`tests/shared/live-warehouse-rows.json`, and the same file is loaded into +DuckDB so the two engines' answers can be compared. + +One fixture it cannot create is the denied table, since an owner can always +read their own relation. Making one needs a second principal, and the +generated SQL says what is left to do. Until its variable is set, the +access-refusal test skips. + +`test_live_warehouse_fixture.py` covers the half of this that needs no +warehouse: the rows load, every corpus definition runs against them, and the +provisioning SQL renders. It runs in CI. diff --git a/pkg-py/tests/_live_warehouse.py b/pkg-py/tests/_live_warehouse.py new file mode 100644 index 000000000..5fce89b42 --- /dev/null +++ b/pkg-py/tests/_live_warehouse.py @@ -0,0 +1,244 @@ +"""Reaching a real Snowflake or Databricks, when one is configured. + +Every other warehouse test in this suite drives a fake that replays canned +rows, which checks how commons reads a listing but not what a warehouse +actually returns. This module is the other half: it builds a SQLAlchemy +engine from the environment and loads the same rows into DuckDB, so the two +engines' answers can be compared. + +The variable names come from `tests/shared/live-warehouse-env.json` rather +than being written here, because the R suite reads the same contract and a +name spelled twice is a name that can drift. Nothing here fails when a +warehouse is absent: an unconfigured backend, or a missing driver package, +skips. + +Bring a warehouse to the expected state with +`scripts/provision-live-warehouse.py`. +""" + +from __future__ import annotations + +import datetime as dt +import math +import os +from dataclasses import dataclass +from typing import Any + +import pytest +import sqlalchemy + +from tests._shared import load_shared_fixture + +__all__ = [ + "LiveWarehouse", + "comparable", + "duckdb_fixture_connection", + "matches", + "require_live_warehouse", +] + +_ENV = load_shared_fixture("live-warehouse-env") +_ROWS = load_shared_fixture("live-warehouse-rows") + +# Doubles JSON cannot spell. The fixture carries them as strings. +_SPECIAL_DOUBLES = {"NaN": math.nan, "Infinity": math.inf, "-Infinity": -math.inf} + + +@dataclass +class LiveWarehouse: + """A configured live backend and the identifiers it was pointed at.""" + + backend: str + engine: sqlalchemy.Engine + namespace: tuple[str, ...] + fixtures: dict[str, str] + + def table_id(self, table: str) -> tuple[str, ...]: + return (*self.namespace, table) + + def dotted(self, table: str) -> str: + return ".".join(self.table_id(table)) + + +def _missing(names: list[str]) -> list[str]: + return [name for name in names if not os.environ.get(name)] + + +def require_live_warehouse(backend: str, *needs: str) -> LiveWarehouse: + """A live backend, or a skip naming what is unset. + + `needs` lists optional fixtures the caller cannot run without, such as + `denied_table`. The required ones are always checked. + """ + spec = _ENV["backends"][backend] + connection = spec["connection"] + + absent = _missing(connection["required"]) + fixtures: dict[str, str] = {} + for name, fixture in spec["fixtures"].items(): + value = os.environ.get(fixture["variable"], "") + if value: + fixtures[name] = value + elif fixture["required"] or name in needs: + absent.append(fixture["variable"]) + if absent: + pytest.skip(f"live {backend} needs {', '.join(sorted(set(absent)))}") + + namespace = tuple(fixtures["schema"].split(".")) + expected = spec["fixtures"]["schema"]["arity"] + if len(namespace) != expected: + raise ValueError( + f"{spec['fixtures']['schema']['variable']} must name " + f"{expected} identifier parts, got {fixtures['schema']!r}." + ) + + engine = _engine(backend, connection, namespace) + return LiveWarehouse( + backend=backend, engine=engine, namespace=namespace, fixtures=fixtures + ) + + +def _engine( + backend: str, connection: dict[str, Any], namespace: tuple[str, ...] +) -> sqlalchemy.Engine: + if backend == "databricks": + pytest.importorskip( + "databricks.sqlalchemy", reason="pip install databricks-sqlalchemy" + ) + host = os.environ["DATABRICKS_HOST"].removeprefix("https://").rstrip("/") + token = os.environ.get(connection["token"], "") + catalog, schema = namespace + query = {"http_path": os.environ["DATABRICKS_HTTP_PATH"]} + url = sqlalchemy.engine.URL.create( + "databricks", + username="token" if token else None, + password=token or None, + host=host, + database=schema, + query={**query, "catalog": catalog}, + ) + # Without a token the connector runs the OAuth user-to-machine flow, + # which opens a browser and reuses the SSO session. That is what makes + # an SSO-only account usable without minting a long-lived secret. + connect_args = {} if token else {"auth_type": "databricks-oauth"} + return sqlalchemy.create_engine(url, connect_args=connect_args) + + pytest.importorskip( + "snowflake.sqlalchemy", reason="pip install snowflake-sqlalchemy" + ) + database, schema = namespace + url = sqlalchemy.engine.URL.create( + "snowflake", + username=os.environ["SNOWFLAKE_USER"], + password=os.environ.get(connection["token"]) or None, + host=os.environ["SNOWFLAKE_ACCOUNT"], + database=f"{database}/{schema}", + query={"warehouse": os.environ["SNOWFLAKE_WAREHOUSE"]}, + ) + return sqlalchemy.create_engine(url) + + +def _duckdb_value(value: Any, type_name: str) -> Any: + """A fixture value as the Python object DuckDB should store.""" + if value is None: + return None + if type_name == "double" and isinstance(value, str): + return _SPECIAL_DOUBLES[value] + if type_name in ("date", "timestamp"): + # Left as the fixture's ISO text and cast in the INSERT. Binding a + # tz-aware datetime makes DuckDB reach for pytz, and the loader should + # not pull a dependency in for a fixture it can hand over as a string. + return value + if type_name == "profile_struct": + return { + "zip": value["zip"], + "geo": {"latitude": value["geo"]["latitude"]} if value["geo"] else None, + "nick names": value["nick names"], + } + return value + + +def duckdb_fixture_connection(): + """The fixture rows in DuckDB, as the reference the warehouse is read against. + + Loaded from the same file the provisioning SQL is generated from, so a row + cannot differ between the two engines and turn a mismatch into a pass. + """ + import duckdb + + con = duckdb.connect() + types = _ROWS["types"] + for table, spec in _ROWS["tables"].items(): + columns = ", ".join( + f'"{column["name"]}" {types[column["type"]]["duckdb"]}' + for column in spec["columns"] + ) + con.execute(f'CREATE TABLE "{table}" ({columns})') + placeholders = ", ".join( + f"CAST(? AS {types[column['type']]['duckdb']})" + for column in spec["columns"] + ) + con.executemany( + f'INSERT INTO "{table}" VALUES ({placeholders})', + [ + [ + _duckdb_value(row[column["name"]], column["type"]) + for column in spec["columns"] + ] + for row in spec["rows"] + ], + ) + return con + + +def comparable(value: Any) -> Any: + """One engine's value in a form the other's can be compared against. + + Drivers disagree about the Python type a column arrives as without + disagreeing about the value: a warehouse double may come back as a + Decimal, and a timestamp's offset may be expressed against a different + zone. Type is not what these tests are about, so it is normalized away + and the value is what remains. + """ + if isinstance(value, dt.datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=dt.UTC) + return value.astimezone(dt.UTC) + if isinstance(value, bool) or value is None: + return value + if isinstance(value, (int, float)) or type(value).__name__ == "Decimal": + return float(value) + if isinstance(value, (list, tuple)): + return [comparable(item) for item in value] + if isinstance(value, dict): + return {key: comparable(item) for key, item in value.items()} + return value + + +def _sort_key(value: Any) -> tuple[str, str]: + return (type(value).__name__, repr(value)) + + +def matches(left: list[Any], right: list[Any]) -> bool: + """Whether two engines produced the same values for one definition. + + Compared as multisets. The fixture tables carry no key, and no ordering + column can be added to them, because the corpus dictionary defines their + columns and a definition asking about every column (`anything missing`) + would then be asking about the key as well. + """ + if len(left) != len(right): + return False + for a, b in zip( + sorted((comparable(v) for v in left), key=_sort_key), + sorted((comparable(v) for v in right), key=_sort_key), + ): + if isinstance(a, float) and isinstance(b, float): + if math.isnan(a) and math.isnan(b): + continue + if math.isclose(a, b, rel_tol=1e-9, abs_tol=1e-12): + continue + return False + if a != b: + return False + return True diff --git a/pkg-py/tests/test_live_warehouse_fixture.py b/pkg-py/tests/test_live_warehouse_fixture.py new file mode 100644 index 000000000..fead2da87 --- /dev/null +++ b/pkg-py/tests/test_live_warehouse_fixture.py @@ -0,0 +1,137 @@ +"""The live-warehouse fixture, checked without a warehouse. + +The tests in `test_live_warehouses.py` skip unless one is configured, which +would leave the rows, the DuckDB loader and the provisioning SQL unexercised +in CI until someone with credentials ran them. These tests cover that half: +the rows load, every corpus definition runs against them, and the generator +renders. What is left needing a real warehouse is only what a warehouse can +answer. +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests._live_warehouse import comparable, duckdb_fixture_connection, matches +from tests._shared import SHARED_DIR, load_shared_fixture +from tests.test_definition_emit_sql import compile_corpus + +ROWS = load_shared_fixture("live-warehouse-rows") +ROWS_UNRUNNABLE = {key for key in ROWS["unrunnable"] if key != "note"} + + +def _generator() -> SimpleNamespace: + """The provisioning script, loaded from the repository root. + + Read and executed rather than imported, because importing it writes a + __pycache__ into scripts/, and a test should not leave anything behind in + a directory of shell scripts. + """ + path = Path(SHARED_DIR).parents[1] / "scripts" / "provision-live-warehouse.py" + namespace: dict[str, object] = {"__file__": str(path), "__name__": "provisioner"} + exec(compile(path.read_text(encoding="utf-8"), str(path), "exec"), namespace) # noqa: S102 + return SimpleNamespace(**namespace) + + +def test_the_fixture_tables_are_the_ones_the_corpus_dictionaries_describe(): + """The rows exist to make the corpus definitions runnable. + + A column added to a dictionary without being added here would leave a + definition referring to something the provisioned table lacks, which + would only surface as a live failure on someone else's machine. + """ + import yaml + + from commons._data_dictionary import DataDictionary + + for table, spec in ROWS["tables"].items(): + path = SHARED_DIR / "definition-export" / "valid" / spec["dictionary"] + dictionary = DataDictionary.model_validate( + yaml.safe_load(path.read_text(encoding="utf-8")) + ) + assert table in dictionary.tables, spec["dictionary"] + authored = set(dictionary.tables[table].columns) + provisioned = {column["name"] for column in spec["columns"]} + assert authored == provisioned, ( + f"{table}: dictionary has {sorted(authored - provisioned)} that the " + f"fixture rows lack, and {sorted(provisioned - authored)} the " + "dictionary does not describe" + ) + + +def test_the_rows_load_into_duckdb_with_their_declared_types(): + con = duckdb_fixture_connection() + + for table, spec in ROWS["tables"].items(): + rows = con.execute(f'SELECT * FROM "{table}"').fetchall() + assert len(rows) == len(spec["rows"]) + assert len(rows) > 5, ( + f"{table} needs more than five rows, so describe_table's sample " + "cap is exercised rather than assumed" + ) + + +def test_every_corpus_definition_runs_against_the_fixture_rows(): + """The DuckDB half of the live comparison, which needs no warehouse. + + A definition that cannot run here would fail live for a reason that has + nothing to do with the warehouse, so this is where that is caught. + """ + con = duckdb_fixture_connection() + + mixed_grain = load_shared_fixture("definitions")["mixed_grain"] + unrunnable = set(ROWS_UNRUNNABLE) + + checked = 0 + expected = 0 + for table, spec in ROWS["tables"].items(): + grain = mixed_grain[spec["dictionary"]] + compiled = compile_corpus(spec["dictionary"], "duckdb") + table_keys = [key for key in compiled if key.startswith(f"{table}::")] + expected += sum( + 1 for key in table_keys if not grain.get(key) and key not in unrunnable + ) + for key, record in compiled.items(): + if not key.startswith(f"{table}::"): + continue + # A mixed-grain definition projects a row-shaped expression over an + # aggregate, so no engine can run it as a bare selection. commons' + # own guard refuses it, and `definitions.json` is where that + # classification lives. + if grain.get(key) or key in unrunnable: + continue + rows = con.execute( + f'SELECT {record.sql} AS result FROM "{table}"' + ).fetchall() + assert rows, key + checked += 1 + assert checked == expected, ( + f"{expected - checked} definitions were dropped without being " + "exempted in the fixture" + ) + assert checked > 35, f"only {checked} definitions ran" + + +@pytest.mark.parametrize("dialect", ["databricks", "snowflake"]) +def test_the_provisioning_script_renders_every_fixture_table(dialect): + sql = _generator().render(dialect, "catalog.schema") + + for table in ROWS["tables"]: + assert "CREATE TABLE" in sql and table in sql + assert sql.count("INSERT INTO") == len(ROWS["tables"]) + # The rows carry doubles JSON cannot spell, and the literal for them is + # per-dialect, so an unrendered one would reach the warehouse as a string. + for literal in _generator().SPECIAL_DOUBLES[dialect].values(): + assert literal in sql, literal + + +def test_a_mismatch_is_reported_as_one(): + """The comparator, since every live assertion rests on it.""" + assert matches([1.0, None], [None, 1.0]) + assert matches([float("nan")], [float("nan")]) + assert not matches([1.0], [1.0, 2.0]) + assert not matches([1.0], [2.0]) + assert comparable(True) is True diff --git a/pkg-py/tests/test_live_warehouses.py b/pkg-py/tests/test_live_warehouses.py new file mode 100644 index 000000000..baa8a5f9c --- /dev/null +++ b/pkg-py/tests/test_live_warehouses.py @@ -0,0 +1,239 @@ +"""What only a real warehouse can tell us. + +Everything else in this suite that touches Snowflake or Databricks runs +against a fake replaying canned rows, so it checks how commons reads a +listing rather than what a warehouse returns. These tests assert the things +the fake cannot answer: the row shapes the listing queries really produce, +how a genuine refusal is reported, what the session identity queries say, +and whether the SQL the definition emitters produce both parses and computes +what DuckDB computes for the same rows. + +Every test skips unless the backend is configured. See +`tests/_live_warehouse.py` for the variables and +`scripts/provision-live-warehouse.py` for the tables. +""" + +from __future__ import annotations + +import pytest + +from commons._backends import EngineBackend +from commons._catalog import ( + CatalogAuthorizationError, + Selector, + _databricks, + _snowflake, + normalize_identifier, + require_queryable, + session_snapshot, +) +from commons._catalog._security import probe_relation +from commons._data_source import TableId, data_source, list_tables +from tests._live_warehouse import ( + duckdb_fixture_connection, + matches, + require_live_warehouse, +) +from tests._shared import load_shared_fixture +from tests.test_definition_emit_sql import compile_corpus + +ROWS_UNRUNNABLE = { + key + for key in load_shared_fixture("live-warehouse-rows")["unrunnable"] + if key != "note" +} + +BACKENDS = ["databricks", "snowflake"] +FIXTURE_TABLES = ["orders", "values", "survey"] + +_READERS = {"databricks": _databricks, "snowflake": _snowflake} +_CASE = {"databricks": "lower", "snowflake": "upper"} + + +def _backend(live) -> EngineBackend: + return EngineBackend(live.engine) + + +def _selector(live) -> Selector: + return Selector(catalog=live.namespace[0], schema=live.namespace[1]) + + +def _folded(live, name: str) -> str: + return normalize_identifier(name, _CASE[live.backend]) + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_the_listing_reports_the_provisioned_relations(backend): + live = require_live_warehouse(backend) + reader = _READERS[backend] + + relations = reader.list_relations(_backend(live), _selector(live)) + + listed = {_folded(live, relation.id.table) for relation in relations} + assert {_folded(live, table) for table in FIXTURE_TABLES} <= listed + for relation in relations: + assert relation.kind in ("table", "view") + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_the_listing_rows_carry_the_keys_the_shared_fixture_pins(backend): + """The shape of a real reply, against the shape the fixture claims. + + `tests/shared/catalog-rows.json` was written from documentation rather + than from a captured reply, so this is the test that turns it from + plausible into checked. Extra keys are allowed, since a warehouse may add + columns to its own listing; a missing one means the fixture is wrong. + """ + live = require_live_warehouse(backend) + pinned = load_shared_fixture("catalog-rows")[backend] + catalog, schema = live.namespace + backend_obj = _backend(live) + + if backend == "databricks": + relation_rows = backend_obj.query( + "SELECT * FROM system.information_schema.tables " + f"WHERE table_catalog = '{catalog}' AND table_schema = '{schema}'" + ) + column_rows = backend_obj.query(f"DESCRIBE TABLE {live.dotted('survey')}") + else: + relation_rows = backend_obj.query( + f'SHOW OBJECTS IN SCHEMA "{catalog}"."{schema}"' + ) + column_rows = backend_obj.query(f"DESC TABLE {live.dotted('survey')}") + + for group, rows in (("relations", relation_rows), ("columns", column_rows)): + assert rows, f"{backend} returned no {group}" + expected = set(pinned[group]["rows"][0]) + actual = {str(key).lower() for key in rows[0]} + assert expected <= actual, ( + f"{backend} {group} rows are missing {sorted(expected - actual)}; " + "tests/shared/catalog-rows.json describes a shape this warehouse " + "does not return" + ) + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_a_source_over_the_namespace_describes_every_fixture_table(backend): + live = require_live_warehouse(backend) + + source = data_source(live.engine, tables=_selector(live)) + + labels = {_folded(live, label) for label in list_tables(source)} + assert {_folded(live, table) for table in FIXTURE_TABLES} <= labels + survey = next( + label + for label in list_tables(source) + if _folded(live, label).endswith("survey") + ) + assert source.relations is not None + columns = source.relations[survey].columns + described = { + str(column.get("col_name") or column.get("name")).lower() for column in columns + } + authored = { + column["name"] + for column in load_shared_fixture("live-warehouse-rows")["tables"]["survey"][ + "columns" + ] + } + assert authored <= described + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_the_session_snapshot_reads_the_live_identity(backend): + live = require_live_warehouse(backend) + + snapshot = session_snapshot(_backend(live)) + + assert snapshot is not None + assert snapshot.principal + # Databricks has no role, which is why `catalog-session-changed.json` + # pins which fields each backend can report as changed at all. + if backend == "databricks": + assert snapshot.role is None + assert snapshot.catalog and snapshot.schema + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_a_refusal_is_classified_as_authorization(backend): + """What a real permission error says, against what the fixture assumed. + + `tests/shared/catalog-access-errors.json` decides from a driver's + SQLSTATE and message whether commons caches the refusal or retries, and + its cases were written from documentation. A refusal this test cannot + classify is a defect in that file, not in the warehouse. + """ + live = require_live_warehouse(backend, "denied_table") + denied = TableId(*live.fixtures["denied_table"].split(".")) + backend_obj = _backend(live) + + probe = probe_relation(backend_obj, denied) + + assert probe.state == "authorization", ( + f"a real {backend} refusal classified as {probe.state!r}: {probe.error}" + ) + with pytest.raises(CatalogAuthorizationError): + require_queryable(backend_obj, denied, denied.label) + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_the_emitted_definition_sql_agrees_with_duckdb(backend): + """Every corpus definition, run on the warehouse and on DuckDB. + + This is what a live run is for. `definition-warehouse-sql.json` pins SQL + no warehouse has ever parsed, so running it proves the syntax; comparing + the values against DuckDB over the same rows proves the emitter did not + merely produce valid SQL for the wrong computation. + + A definition whose record carries notes is exempt from the comparison and + only has to execute. The notes are where an emitter records a deliberate + divergence from data-dict's semantics (trim, LIKE escaping, regex + dialect), so they name exactly the cases whose answers are expected to + differ. Reading the exemption off the notes rather than off a list here + means a divergence cannot be claimed without also being documented. + """ + live = require_live_warehouse(backend) + duckdb_con = duckdb_fixture_connection() + rows_fixture = load_shared_fixture("live-warehouse-rows") + backend_obj = _backend(live) + + mixed_grain = load_shared_fixture("definitions")["mixed_grain"] + unrunnable = set(ROWS_UNRUNNABLE) + + compared = 0 + executed = 0 + for table, spec in rows_fixture["tables"].items(): + warehouse_records = compile_corpus(spec["dictionary"], backend) + duckdb_records = compile_corpus(spec["dictionary"], "duckdb") + grain = mixed_grain[spec["dictionary"]] + for key, record in warehouse_records.items(): + if not key.startswith(f"{table}::"): + continue + # Projecting a row-shaped expression over an aggregate is what + # commons' own grain guard refuses, so no engine can run it as a + # bare selection and there is nothing to compare. + if grain.get(key) or key in unrunnable: + continue + warehouse_values = [ + next(iter(row.values())) + for row in backend_obj.query( + f"SELECT {record.sql} AS result FROM {live.dotted(table)}" + ) + ] + executed += 1 + if record.notes: + continue + expected = [ + row[0] + for row in duckdb_con.execute( + f'SELECT {duckdb_records[key].sql} AS result FROM "{table}"' + ).fetchall() + ] + assert matches(warehouse_values, expected), ( + f"{key} on {backend} returned {warehouse_values!r}, DuckDB " + f"returned {expected!r}, and the record documents no divergence" + ) + compared += 1 + + assert executed, "no definitions were executed" + assert compared, "every definition was exempted, so nothing was compared" diff --git a/pkg-py/uv.lock b/pkg-py/uv.lock index 89280ade5..aabf96dae 100644 --- a/pkg-py/uv.lock +++ b/pkg-py/uv.lock @@ -585,11 +585,16 @@ dev = [ { name = "pyrefly" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytz" }, { name = "ruff" }, ] docs = [ { name = "great-docs" }, ] +live-warehouse = [ + { name = "databricks-sqlalchemy" }, + { name = "snowflake-sqlalchemy" }, +] [package.metadata] requires-dist = [ @@ -626,9 +631,14 @@ dev = [ { name = "pyrefly" }, { name = "pytest", specifier = ">=8" }, { name = "pytest-asyncio" }, + { name = "pytz" }, { name = "ruff" }, ] docs = [{ name = "great-docs", specifier = ">=0.17" }] +live-warehouse = [ + { name = "databricks-sqlalchemy", specifier = ">=2" }, + { name = "snowflake-sqlalchemy", specifier = ">=1.6" }, +] [[package]] name = "contourpy" @@ -811,6 +821,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/bf/eb4ff492e1c3d027617b5a20e4e6d63a035b41b8cf77a95950159556a3a5/contourpy-1.4.0-cp315-cp315t-win_arm64.whl", hash = "sha256:78ed5c5f962b3e156109f0531b174a65ba80d13cc681a70227b12524c162e184", size = 587064, upload-time = "2026-09-11T19:04:51.148Z" }, ] +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/ce/cb/52eb3770c0d0be2702a98c6e96065ddc0a2877cf0845aa9c23397c142cd4/cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8", size = 3892731, upload-time = "2026-09-30T14:44:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/19/8e/aa1fc533d4546b127b45de8aa024eb5933d23eff9debfe25931e56861095/cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047", size = 4710431, upload-time = "2026-09-30T14:44:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/6a/64/72bc3f75176e7e406b748a3e3830432b8c51297b38368713df04dc04898a/cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539", size = 4694824, upload-time = "2026-09-30T14:44:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c6/62c77550edfa5ca3f14bf44a1e6739b9fa09d6e998a11d97ed8213bccc98/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1", size = 4716967, upload-time = "2026-09-30T14:44:19.661Z" }, + { url = "https://files.pythonhosted.org/packages/f4/37/cce70f150c432914460157a6ecc161752e053aa5ec0ef3b3f7dc6e31039a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7", size = 5328676, upload-time = "2026-09-30T14:44:21.744Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9a/6f2f0304d634ceafdeaf23e84537336664ac419b5d07611675c2ad3f6b7a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18", size = 4727698, upload-time = "2026-09-30T14:44:24.178Z" }, + { url = "https://files.pythonhosted.org/packages/1d/de/66bcf9244d118663b2e1aaded8990f4640e3d7b7411870a5765f252074d2/cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37", size = 4354821, upload-time = "2026-09-30T14:44:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e6/db28a28c7b6c676addce89136de3d8db49ea825a8c863472e36e42ead4ad/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2", size = 4716748, upload-time = "2026-09-30T14:44:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/30/96/01546c7f69ea0e2ab790a2e4f0934a4052fb9b388147fbf83c2fd72f1e57/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1", size = 5285085, upload-time = "2026-09-30T14:44:30.704Z" }, + { url = "https://files.pythonhosted.org/packages/6c/01/03263395f74d50b071e9e66daace3f8bef80493e5d410726f2ba8554736b/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05", size = 4727268, upload-time = "2026-09-30T14:44:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/2bfe8f29ec0cc9c0d99359c4161adf32858e4934b72c6d100d2ac0bbe962/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e", size = 4849503, upload-time = "2026-09-30T14:44:34.969Z" }, + { url = "https://files.pythonhosted.org/packages/54/44/e80651ecbf0e42b62e2bb5f5768916e07eea72e1297338956a61df361f88/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e", size = 5004057, upload-time = "2026-09-30T14:44:37.064Z" }, + { url = "https://files.pythonhosted.org/packages/f8/cc/1d33befb3cd7ea7e77d2d73f43f2066471da1b21f24a6156efcaabf6d2e8/cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45", size = 3795868, upload-time = "2026-09-30T14:44:39.71Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, + { url = "https://files.pythonhosted.org/packages/1d/7a/f08d34ce09d60f89ebd391e2ebc6ba2b995e6dd7552f41820f8085f94e53/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67", size = 4716035, upload-time = "2026-09-30T15:29:48.681Z" }, + { url = "https://files.pythonhosted.org/packages/45/67/e18fb65592451a2acb76e9f2fbe14e0f47a8318b4c5430f1633851d03daa/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a", size = 4726917, upload-time = "2026-09-30T15:29:50.608Z" }, + { url = "https://files.pythonhosted.org/packages/83/28/38fdce17e60f6b825e69fc3b7f75e70a6612759980704697e1de4cbfaf6e/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48", size = 4715341, upload-time = "2026-09-30T15:29:52.522Z" }, + { url = "https://files.pythonhosted.org/packages/b6/b1/d9121a717e0f893c64bd6ca7702614778d7df2a5c309128a002421788516/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42", size = 4726322, upload-time = "2026-09-30T15:29:54.263Z" }, + { url = "https://files.pythonhosted.org/packages/36/8b/e6d153808bf353e152abd2fd4d8f09670d956ac78379ac46e60d7efbf04c/cryptography-50.0.2-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81", size = 3873097, upload-time = "2026-09-30T15:29:56.097Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1d/1271f287ff7170ddafc2aad36260c4eec20ccd2fea70f38455e9d56d427b/cryptography-50.0.2-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452", size = 3805376, upload-time = "2026-09-30T15:29:58.729Z" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -829,6 +908,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/84/582e1daf3f2eb296573c11b774abf10223e9858d4c5c5c936180b643a720/databackend-0.0.3-py3-none-any.whl", hash = "sha256:2ded97ff85a0ca272d752fbc0513f70512d2c23325e3c72de15613df7e959ef5", size = 7001, upload-time = "2024-08-21T21:09:41.137Z" }, ] +[[package]] +name = "databricks-sql-connector" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lz4" }, + { name = "oauthlib" }, + { name = "openpyxl" }, + { name = "pandas" }, + { name = "pybreaker" }, + { name = "pyjwt" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "thrift" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/48/13691ccdf268aaddea0d981563616ad5f0811d8626a45a3d061de50a9174/databricks_sql_connector-4.6.0.tar.gz", hash = "sha256:2c3fdb28a08b7e9d4c10775f88aa7d6cf8212a1f88f5baacb05f294ee148c1e5", size = 244176, upload-time = "2026-09-24T18:30:55.003Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/4f/198110284fca731db1b7876b206138ee81b92bbf8d10f4c024cfd00e8b7d/databricks_sql_connector-4.6.0-py3-none-any.whl", hash = "sha256:9f2128592052d3aadb7a3c46df0a8ef414402ec6ea7a0fdb8490da737fe43cdc", size = 270341, upload-time = "2026-09-24T18:30:53.306Z" }, +] + +[[package]] +name = "databricks-sqlalchemy" +version = "2.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "databricks-sql-connector" }, + { name = "pyarrow" }, + { name = "sqlalchemy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/ef/04ed94460f6be166f799aa00cade8fc76b718994a11e9857ab7c19ce17db/databricks_sqlalchemy-2.0.10.tar.gz", hash = "sha256:75e75b1309a6cd9b8442060523f1241254766d90c130b6fc9575a0e2dd692e39", size = 32636, upload-time = "2026-07-02T21:20:39.084Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/70/6915ef703957d32bb4c7ff21e39da56ce96f92d7962078b240507d72aacc/databricks_sqlalchemy-2.0.10-py3-none-any.whl", hash = "sha256:47ad32cc8660bf11fbe1ad8065f976264f7d2399e517144518e77fd6f5ba90b4", size = 32913, upload-time = "2026-07-02T21:20:37.896Z" }, +] + [[package]] name = "debugpy" version = "1.8.21" @@ -899,6 +1013,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -2028,6 +2151,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/d4/1152d1c7ab42d8b908be64fd200ddc870dc9d4925e951198702084aa1a7d/linkify_it_py-2.2.0-py3-none-any.whl", hash = "sha256:3adc40eb5af300b2605fcfdb968c24e1d780a90f1f2221af7c15e5111e94d443", size = 21971, upload-time = "2026-08-29T07:07:07.164Z" }, ] +[[package]] +name = "lz4" +version = "4.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/5b/6edcd23319d9e28b1bedf32768c3d1fd56eed8223960a2c47dacd2cec2af/lz4-4.4.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d6da84a26b3aa5da13a62e4b89ab36a396e9327de8cd48b436a3467077f8ccd4", size = 207391, upload-time = "2025-11-03T13:01:36.644Z" }, + { url = "https://files.pythonhosted.org/packages/34/36/5f9b772e85b3d5769367a79973b8030afad0d6b724444083bad09becd66f/lz4-4.4.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61d0ee03e6c616f4a8b69987d03d514e8896c8b1b7cc7598ad029e5c6aedfd43", size = 207146, upload-time = "2025-11-03T13:01:37.928Z" }, + { url = "https://files.pythonhosted.org/packages/04/f4/f66da5647c0d72592081a37c8775feacc3d14d2625bbdaabd6307c274565/lz4-4.4.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:33dd86cea8375d8e5dd001e41f321d0a4b1eb7985f39be1b6a4f466cd480b8a7", size = 1292623, upload-time = "2025-11-03T13:01:39.341Z" }, + { url = "https://files.pythonhosted.org/packages/85/fc/5df0f17467cdda0cad464a9197a447027879197761b55faad7ca29c29a04/lz4-4.4.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:609a69c68e7cfcfa9d894dc06be13f2e00761485b62df4e2472f1b66f7b405fb", size = 1279982, upload-time = "2025-11-03T13:01:40.816Z" }, + { url = "https://files.pythonhosted.org/packages/25/3b/b55cb577aa148ed4e383e9700c36f70b651cd434e1c07568f0a86c9d5fbb/lz4-4.4.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:75419bb1a559af00250b8f1360d508444e80ed4b26d9d40ec5b09fe7875cb989", size = 1368674, upload-time = "2025-11-03T13:01:42.118Z" }, + { url = "https://files.pythonhosted.org/packages/fb/31/e97e8c74c59ea479598e5c55cbe0b1334f03ee74ca97726e872944ed42df/lz4-4.4.5-cp311-cp311-win32.whl", hash = "sha256:12233624f1bc2cebc414f9efb3113a03e89acce3ab6f72035577bc61b270d24d", size = 88168, upload-time = "2025-11-03T13:01:43.282Z" }, + { url = "https://files.pythonhosted.org/packages/18/47/715865a6c7071f417bef9b57c8644f29cb7a55b77742bd5d93a609274e7e/lz4-4.4.5-cp311-cp311-win_amd64.whl", hash = "sha256:8a842ead8ca7c0ee2f396ca5d878c4c40439a527ebad2b996b0444f0074ed004", size = 99491, upload-time = "2025-11-03T13:01:44.167Z" }, + { url = "https://files.pythonhosted.org/packages/14/e7/ac120c2ca8caec5c945e6356ada2aa5cfabd83a01e3170f264a5c42c8231/lz4-4.4.5-cp311-cp311-win_arm64.whl", hash = "sha256:83bc23ef65b6ae44f3287c38cbf82c269e2e96a26e560aa551735883388dcc4b", size = 91271, upload-time = "2025-11-03T13:01:45.016Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ac/016e4f6de37d806f7cc8f13add0a46c9a7cfc41a5ddc2bc831d7954cf1ce/lz4-4.4.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:df5aa4cead2044bab83e0ebae56e0944cc7fcc1505c7787e9e1057d6d549897e", size = 207163, upload-time = "2025-11-03T13:01:45.895Z" }, + { url = "https://files.pythonhosted.org/packages/8d/df/0fadac6e5bd31b6f34a1a8dbd4db6a7606e70715387c27368586455b7fc9/lz4-4.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d0bf51e7745484d2092b3a51ae6eb58c3bd3ce0300cf2b2c14f76c536d5697a", size = 207150, upload-time = "2025-11-03T13:01:47.205Z" }, + { url = "https://files.pythonhosted.org/packages/b7/17/34e36cc49bb16ca73fb57fbd4c5eaa61760c6b64bce91fcb4e0f4a97f852/lz4-4.4.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7b62f94b523c251cf32aa4ab555f14d39bd1a9df385b72443fd76d7c7fb051f5", size = 1292045, upload-time = "2025-11-03T13:01:48.667Z" }, + { url = "https://files.pythonhosted.org/packages/90/1c/b1d8e3741e9fc89ed3b5f7ef5f22586c07ed6bb04e8343c2e98f0fa7ff04/lz4-4.4.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c3ea562c3af274264444819ae9b14dbbf1ab070aff214a05e97db6896c7597e", size = 1279546, upload-time = "2025-11-03T13:01:50.159Z" }, + { url = "https://files.pythonhosted.org/packages/55/d9/e3867222474f6c1b76e89f3bd914595af69f55bf2c1866e984c548afdc15/lz4-4.4.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24092635f47538b392c4eaeff14c7270d2c8e806bf4be2a6446a378591c5e69e", size = 1368249, upload-time = "2025-11-03T13:01:51.273Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e7/d667d337367686311c38b580d1ca3d5a23a6617e129f26becd4f5dc458df/lz4-4.4.5-cp312-cp312-win32.whl", hash = "sha256:214e37cfe270948ea7eb777229e211c601a3e0875541c1035ab408fbceaddf50", size = 88189, upload-time = "2025-11-03T13:01:52.605Z" }, + { url = "https://files.pythonhosted.org/packages/a5/0b/a54cd7406995ab097fceb907c7eb13a6ddd49e0b231e448f1a81a50af65c/lz4-4.4.5-cp312-cp312-win_amd64.whl", hash = "sha256:713a777de88a73425cf08eb11f742cd2c98628e79a8673d6a52e3c5f0c116f33", size = 99497, upload-time = "2025-11-03T13:01:53.477Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7e/dc28a952e4bfa32ca16fa2eb026e7a6ce5d1411fcd5986cd08c74ec187b9/lz4-4.4.5-cp312-cp312-win_arm64.whl", hash = "sha256:a88cbb729cc333334ccfb52f070463c21560fca63afcf636a9f160a55fac3301", size = 91279, upload-time = "2025-11-03T13:01:54.419Z" }, + { url = "https://files.pythonhosted.org/packages/2f/46/08fd8ef19b782f301d56a9ccfd7dafec5fd4fc1a9f017cf22a1accb585d7/lz4-4.4.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6bb05416444fafea170b07181bc70640975ecc2a8c92b3b658c554119519716c", size = 207171, upload-time = "2025-11-03T13:01:56.595Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3f/ea3334e59de30871d773963997ecdba96c4584c5f8007fd83cfc8f1ee935/lz4-4.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b424df1076e40d4e884cfcc4c77d815368b7fb9ebcd7e634f937725cd9a8a72a", size = 207163, upload-time = "2025-11-03T13:01:57.721Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/7b3a2a0feb998969f4793c650bb16eff5b06e80d1f7bff867feb332f2af2/lz4-4.4.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:216ca0c6c90719731c64f41cfbd6f27a736d7e50a10b70fad2a9c9b262ec923d", size = 1292136, upload-time = "2025-11-03T13:02:00.375Z" }, + { url = "https://files.pythonhosted.org/packages/89/d1/f1d259352227bb1c185288dd694121ea303e43404aa77560b879c90e7073/lz4-4.4.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:533298d208b58b651662dd972f52d807d48915176e5b032fb4f8c3b6f5fe535c", size = 1279639, upload-time = "2025-11-03T13:02:01.649Z" }, + { url = "https://files.pythonhosted.org/packages/d2/fb/ba9256c48266a09012ed1d9b0253b9aa4fe9cdff094f8febf5b26a4aa2a2/lz4-4.4.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:451039b609b9a88a934800b5fc6ee401c89ad9c175abf2f4d9f8b2e4ef1afc64", size = 1368257, upload-time = "2025-11-03T13:02:03.35Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6d/dee32a9430c8b0e01bbb4537573cabd00555827f1a0a42d4e24ca803935c/lz4-4.4.5-cp313-cp313-win32.whl", hash = "sha256:a5f197ffa6fc0e93207b0af71b302e0a2f6f29982e5de0fbda61606dd3a55832", size = 88191, upload-time = "2025-11-03T13:02:04.406Z" }, + { url = "https://files.pythonhosted.org/packages/18/e0/f06028aea741bbecb2a7e9648f4643235279a770c7ffaf70bd4860c73661/lz4-4.4.5-cp313-cp313-win_amd64.whl", hash = "sha256:da68497f78953017deb20edff0dba95641cc86e7423dfadf7c0264e1ac60dc22", size = 99502, upload-time = "2025-11-03T13:02:05.886Z" }, + { url = "https://files.pythonhosted.org/packages/61/72/5bef44afb303e56078676b9f2486f13173a3c1e7f17eaac1793538174817/lz4-4.4.5-cp313-cp313-win_arm64.whl", hash = "sha256:c1cfa663468a189dab510ab231aad030970593f997746d7a324d40104db0d0a9", size = 91285, upload-time = "2025-11-03T13:02:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/6a5c2952971af73f15ed4ebfdd69774b454bd0dc905b289082ca8664fba1/lz4-4.4.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:67531da3b62f49c939e09d56492baf397175ff39926d0bd5bd2d191ac2bff95f", size = 207348, upload-time = "2025-11-03T13:02:08.117Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d7/fd62cbdbdccc35341e83aabdb3f6d5c19be2687d0a4eaf6457ddf53bba64/lz4-4.4.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a1acbbba9edbcbb982bc2cac5e7108f0f553aebac1040fbec67a011a45afa1ba", size = 207340, upload-time = "2025-11-03T13:02:09.152Z" }, + { url = "https://files.pythonhosted.org/packages/77/69/225ffadaacb4b0e0eb5fd263541edd938f16cd21fe1eae3cd6d5b6a259dc/lz4-4.4.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a482eecc0b7829c89b498fda883dbd50e98153a116de612ee7c111c8bcf82d1d", size = 1293398, upload-time = "2025-11-03T13:02:10.272Z" }, + { url = "https://files.pythonhosted.org/packages/c6/9e/2ce59ba4a21ea5dc43460cba6f34584e187328019abc0e66698f2b66c881/lz4-4.4.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e099ddfaa88f59dd8d36c8a3c66bd982b4984edf127eb18e30bb49bdba68ce67", size = 1281209, upload-time = "2025-11-03T13:02:12.091Z" }, + { url = "https://files.pythonhosted.org/packages/80/4f/4d946bd1624ec229b386a3bc8e7a85fa9a963d67d0a62043f0af0978d3da/lz4-4.4.5-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2af2897333b421360fdcce895c6f6281dc3fab018d19d341cf64d043fc8d90d", size = 1369406, upload-time = "2025-11-03T13:02:13.683Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/d429ba4720a9064722698b4b754fb93e42e625f1318b8fe834086c7c783b/lz4-4.4.5-cp313-cp313t-win32.whl", hash = "sha256:66c5de72bf4988e1b284ebdd6524c4bead2c507a2d7f172201572bac6f593901", size = 88325, upload-time = "2025-11-03T13:02:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/7ba10c9b97c06af6c8f7032ec942ff127558863df52d866019ce9d2425cf/lz4-4.4.5-cp313-cp313t-win_amd64.whl", hash = "sha256:cdd4bdcbaf35056086d910d219106f6a04e1ab0daa40ec0eeef1626c27d0fddb", size = 99643, upload-time = "2025-11-03T13:02:15.978Z" }, + { url = "https://files.pythonhosted.org/packages/77/4d/a175459fb29f909e13e57c8f475181ad8085d8d7869bd8ad99033e3ee5fa/lz4-4.4.5-cp313-cp313t-win_arm64.whl", hash = "sha256:28ccaeb7c5222454cd5f60fcd152564205bcb801bd80e125949d2dfbadc76bbd", size = 91504, upload-time = "2025-11-03T13:02:17.313Z" }, + { url = "https://files.pythonhosted.org/packages/63/9c/70bdbdb9f54053a308b200b4678afd13efd0eafb6ddcbb7f00077213c2e5/lz4-4.4.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c216b6d5275fc060c6280936bb3bb0e0be6126afb08abccde27eed23dead135f", size = 207586, upload-time = "2025-11-03T13:02:18.263Z" }, + { url = "https://files.pythonhosted.org/packages/b6/cb/bfead8f437741ce51e14b3c7d404e3a1f6b409c440bad9b8f3945d4c40a7/lz4-4.4.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c8e71b14938082ebaf78144f3b3917ac715f72d14c076f384a4c062df96f9df6", size = 207161, upload-time = "2025-11-03T13:02:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/e7/18/b192b2ce465dfbeabc4fc957ece7a1d34aded0d95a588862f1c8a86ac448/lz4-4.4.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b5e6abca8df9f9bdc5c3085f33ff32cdc86ed04c65e0355506d46a5ac19b6e9", size = 1292415, upload-time = "2025-11-03T13:02:20.829Z" }, + { url = "https://files.pythonhosted.org/packages/67/79/a4e91872ab60f5e89bfad3e996ea7dc74a30f27253faf95865771225ccba/lz4-4.4.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b84a42da86e8ad8537aabef062e7f661f4a877d1c74d65606c49d835d36d668", size = 1279920, upload-time = "2025-11-03T13:02:22.013Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/d52c7b11eaa286d49dae619c0eec4aabc0bf3cda7a7467eb77c62c4471f3/lz4-4.4.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bba042ec5a61fa77c7e380351a61cb768277801240249841defd2ff0a10742f", size = 1368661, upload-time = "2025-11-03T13:02:23.208Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/137ddeea14c2cb86864838277b2607d09f8253f152156a07f84e11768a28/lz4-4.4.5-cp314-cp314-win32.whl", hash = "sha256:bd85d118316b53ed73956435bee1997bd06cc66dd2fa74073e3b1322bd520a67", size = 90139, upload-time = "2025-11-03T13:02:24.301Z" }, + { url = "https://files.pythonhosted.org/packages/18/2c/8332080fd293f8337779a440b3a143f85e374311705d243439a3349b81ad/lz4-4.4.5-cp314-cp314-win_amd64.whl", hash = "sha256:92159782a4502858a21e0079d77cdcaade23e8a5d252ddf46b0652604300d7be", size = 101497, upload-time = "2025-11-03T13:02:25.187Z" }, + { url = "https://files.pythonhosted.org/packages/ca/28/2635a8141c9a4f4bc23f5135a92bbcf48d928d8ca094088c962df1879d64/lz4-4.4.5-cp314-cp314-win_arm64.whl", hash = "sha256:d994b87abaa7a88ceb7a37c90f547b8284ff9da694e6afcfaa8568d739faf3f7", size = 93812, upload-time = "2025-11-03T13:02:26.133Z" }, +] + [[package]] name = "magika" version = "0.6.2" @@ -2649,6 +2820,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + [[package]] name = "onnxruntime" version = "1.20.1" @@ -2744,6 +2924,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/db/2b7a1b3de659bb82aef979116c74e809982b13e42c057759767552b5155f/openai-3.3.1-py3-none-any.whl", hash = "sha256:9652df7fdf8ee6f5bd58e0a12f2b1d414a18e0f06bb7a9a57c8643a5f5469bd3", size = 1690337, upload-time = "2026-08-19T16:31:32.812Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.44.0" @@ -3196,17 +3388,17 @@ wheels = [ [[package]] name = "protobuf" -version = "7.36.0" +version = "6.33.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/e7/0553e21d25ca4d9f573135775348a372c3ec34a93a71d5f297c3bac38341/protobuf-7.36.0.tar.gz", hash = "sha256:e8e09cb0d794c6687926fa558a8a6e72aa10edb997d5ca61da0765f12a3e00ea", size = 510034, upload-time = "2026-08-20T16:34:01.071Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/ae/58e3ca96cb2e118cc546b677359b3c6659f79a140935c08dec94c7998585/protobuf-7.36.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:9103532dffd80c6fab7e50c65a31007680a06eb57537d437bb1b35812c138a37", size = 453256, upload-time = "2026-08-20T16:33:53.945Z" }, - { url = "https://files.pythonhosted.org/packages/f0/15/5162230af4912697f0fe406f6800f80760945babcff0e2c2fe6c84ef2d5d/protobuf-7.36.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:bf94a5917c71058262de683669bc0a797a7669d3de71f0b36d058e3194f47b44", size = 341436, upload-time = "2026-08-20T16:33:55.134Z" }, - { url = "https://files.pythonhosted.org/packages/d7/09/1670b2bfc9a45e807e520c3e9be36524db9ccc7dc05ea17af7681cabdc61/protobuf-7.36.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:3297e60abdff301e5f74393d87f6cc59dacab5f024a89548a6e8de1d26576b16", size = 354440, upload-time = "2026-08-20T16:33:56.077Z" }, - { url = "https://files.pythonhosted.org/packages/c7/f8/bd5804695ba400e423c33fd4d9f58c28d86633d5ba1945c36ff3967d98cb/protobuf-7.36.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:70f5ec8eb0da81a44360c0dc0beac99a0d78071d21956a7076bae8bd2051841b", size = 340439, upload-time = "2026-08-20T16:33:56.992Z" }, - { url = "https://files.pythonhosted.org/packages/ef/9f/acd02338235a3e7d03168c4303478347b7624fc8189ff4e7f0d2654bbe86/protobuf-7.36.0-cp310-abi3-win32.whl", hash = "sha256:7326fd717bdc419162a735938d89d4032332bcc3408804012b24ff3a37086071", size = 440216, upload-time = "2026-08-20T16:33:57.99Z" }, - { url = "https://files.pythonhosted.org/packages/0e/4e/12cb93270967a2affff5b3f720694700d4d87712a67afd05c8cb3f6fa52c/protobuf-7.36.0-cp310-abi3-win_amd64.whl", hash = "sha256:1781cc1de61249b750848029bca452c0a8b7e990080316b9bbc2518b2117b488", size = 453731, upload-time = "2026-08-20T16:33:58.951Z" }, - { url = "https://files.pythonhosted.org/packages/01/c3/629999e78d46c1115c11886d51c6bd68c17ce4a944f1ea3e153a91316a33/protobuf-7.36.0-py3-none-any.whl", hash = "sha256:53374d53fc29a67f7dbbf0ade47d7526a0f0137bf0f9c90e48d8a60790ef748c", size = 177024, upload-time = "2026-08-20T16:34:00.053Z" }, + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] [[package]] @@ -3341,6 +3533,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/c0/37d4a7e8e2f7a6076283673d5298018ca26478b934c6ee369e10505ab32c/pyarrow-25.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4288f27577352d608ca08553b0865e4a9b3aa14820c5d95b53337218d609835b", size = 28753071, upload-time = "2026-08-10T12:40:46.623Z" }, ] +[[package]] +name = "pybreaker" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/89/fbf98e383f1ec6d117af2cd983efdb3eb7018b63834c427025764194cac2/pybreaker-1.4.1.tar.gz", hash = "sha256:8df2d245c73ba40c8242c56ffb4f12138fbadc23e296224740c2028ea9dc1178", size = 15555, upload-time = "2025-09-21T15:12:04.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/75/e64d3d40a741e2be21d69154f4e5c43a66f0c603c5ef11f49e01429a5932/pybreaker-1.4.1-py3-none-any.whl", hash = "sha256:b4dab4a05195b7f2a64a6c1a6c4ba7a96534ef56ea7210e6bcb59f28897160e0", size = 12915, upload-time = "2025-09-21T15:12:02.284Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -3476,6 +3677,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] +[[package]] +name = "pyjwt" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, +] + [[package]] name = "pyparsing" version = "3.3.3" @@ -3581,6 +3791,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] +[[package]] +name = "pytz" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/21/d83d6ef28c4c912c4bb4d1dcf591f7b8c6bde87b9c66f9f454677314e16d/pytz-2026.5.tar.gz", hash = "sha256:fa23724b9c486543b9ff54a327ee7569ac83ade54bb9afd0fc18676620401c86", size = 318572, upload-time = "2026-10-04T02:37:58.719Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/ef/c66110d46fb800dda0bf33164182dfadabe26a90e4476844d502a23dca8e/pytz-2026.5-py2.py3-none-any.whl", hash = "sha256:e658af3757f9e26a9d25dd2aff38335acd92bc9104f890a894b2c1ba28311b03", size = 506342, upload-time = "2026-10-04T02:37:56.814Z" }, +] + [[package]] name = "pywinpty" version = "3.0.5" @@ -4219,6 +4438,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "snowflake-connector-python" +version = "5.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "protobuf" }, + { name = "pytz" }, + { name = "tomlkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/b9/2da3619b66e0a5225a7c63a44ab93337c92f85b880fa4f8f2ff1eeedb089/snowflake_connector_python-5.0.0.tar.gz", hash = "sha256:94677d66208a3b1626324e0b9312b577cab99f6431b21aee19b24c9ec9acf648", size = 1852374, upload-time = "2026-10-15T11:04:04.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/19/5e84de489fb7014e7bda9857db66637c9e6c69203ab82ec83778e06d6ce6/snowflake_connector_python-5.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:aa37f23e46a939d8ae97def93c224ab88d69ceee0ce590ef0a43304f4667de30", size = 11049953, upload-time = "2026-10-15T11:03:21.763Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ac/8bbf76798e76e4b2dc8714449246d46a6778e0f1230bcda98c7630ff3dd1/snowflake_connector_python-5.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff1280242a77a0362fb057213c1bc071eae90e6ceabc505e0b299301ca563bc7", size = 10430532, upload-time = "2026-10-15T11:03:11.642Z" }, + { url = "https://files.pythonhosted.org/packages/55/49/c20d27b64f59e31716fe997794c8447b05d83f35fafd315db55b8756993e/snowflake_connector_python-5.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0a33770621f29a2b37d9b7eb20060300c02368ab09cd4ad78aebea2b799c83e4", size = 12414561, upload-time = "2026-10-15T11:03:40.656Z" }, + { url = "https://files.pythonhosted.org/packages/aa/96/a14b0dc3dcdeba256c6d286c9a233e1460153d8d147dda773e573378dc12/snowflake_connector_python-5.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8f468c9b0f1433a41b5ec46f030a966569f40b2398825a3733b3ef224b12ffef", size = 13370451, upload-time = "2026-10-15T11:03:30.467Z" }, + { url = "https://files.pythonhosted.org/packages/c1/60/2fc246a60bee678baaf111bb9a3555dacc4dcc976e5410f36f66c0a84a6c/snowflake_connector_python-5.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:9d118f8809cb640c50f36ddac404140c36bbbc0b4b7c715273cb5fd18cb4d940", size = 11475012, upload-time = "2026-10-15T11:03:50.544Z" }, + { url = "https://files.pythonhosted.org/packages/05/49/95509088aa0546ede0950c639e8703c86e082e5e11285faa71b8cb2f69f1/snowflake_connector_python-5.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:b557caddbfdd2cc0eb65ce2db415f09ddbb5ffc7cc2bcb75aead466ace7c5fa2", size = 10880613, upload-time = "2026-10-15T11:03:59.754Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7e/48b05f1600c7185e28e9455e023b50fbccaf47e9682360622cb5db0008c3/snowflake_connector_python-5.0.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:7042b58fceaffcebf21562e4144fe3c22fbe5626f3bcde8b47236468818fb908", size = 11226471, upload-time = "2026-10-15T11:03:24.085Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/f7da1efc081123c8af0e65d1946b407512d10dfe37e685222107ef552031/snowflake_connector_python-5.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2326fa7ae44d5d9163d70bb82e12317e9864413fd8dac206203b47b1ba7e2e09", size = 10578310, upload-time = "2026-10-15T11:03:14.765Z" }, + { url = "https://files.pythonhosted.org/packages/49/51/7a33757c27cbd2d9c0de75df5f627ea7cedc6335292535611ea394b6ec43/snowflake_connector_python-5.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ac7930d970707251ef6646eb98ef8e836c404642c20dc0d15d1597962b2597bc", size = 14186734, upload-time = "2026-10-15T11:03:43.13Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0e/4b5eef7c6c99fa781aa40599a8052b94740b6b0ca55ee04960bf5754d79f/snowflake_connector_python-5.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a97160c98d9ddecaf8329a2223f15691566ba681e8b3a7fd827ee9c25e473287", size = 15190111, upload-time = "2026-10-15T11:03:32.904Z" }, + { url = "https://files.pythonhosted.org/packages/45/9d/09a201949455c5f580d286c41b364f2eb9fe9375b07f1ea87f5661de3811/snowflake_connector_python-5.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:724d58959aef18afc4c7075161488da8ddde22fbb4fa82190a53ea21c4f63cb5", size = 11560768, upload-time = "2026-10-15T11:03:52.97Z" }, + { url = "https://files.pythonhosted.org/packages/b9/56/cb5a5805086ae2fa4fa6ea9e454d8a66216cc2aeb40fc4fadcbcdc087069/snowflake_connector_python-5.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fdfbc3078d35d7bbb7b6054ac0954915acaccbc64cdceb97a7f221e90e934d9", size = 10876985, upload-time = "2026-10-15T11:04:01.817Z" }, + { url = "https://files.pythonhosted.org/packages/95/f4/7b6530a8b844917eadc59bc1686eb09f0c697197d8d340b36b4c9d909635/snowflake_connector_python-5.0.0-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:8216b006ea5c08f4ae1e14c4440db8f4881c671690f73c89027f81ccb2f0cc0e", size = 11390176, upload-time = "2026-10-15T11:03:26.088Z" }, + { url = "https://files.pythonhosted.org/packages/d0/3d/c2f1b825798c7accb4e6d30d5622feb710ab0419737cf8f5ad59e5aa5ecb/snowflake_connector_python-5.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:33303d157f2457f439895ae20485f7b93e7e23b4bfc13f714c3dcb21b9f5edb3", size = 10731378, upload-time = "2026-10-15T11:03:17.582Z" }, + { url = "https://files.pythonhosted.org/packages/43/73/29234b82a1d96845ec87dbbea5032df23bc995db53543bf05ec7b060090b/snowflake_connector_python-5.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f200fd88b3255c649a1eaf7f2c24f7b5b5e93f2b25020ab03d6b8d055200514e", size = 15954164, upload-time = "2026-10-15T11:03:45.578Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d9/b81b486b252bf70ce55184357311a0e744f4d4b8c436402963768a60de90/snowflake_connector_python-5.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:071a3021149fda531e426157dab0d137c772f5cdb89d03516a3aac7519261837", size = 17009082, upload-time = "2026-10-15T11:03:35.606Z" }, + { url = "https://files.pythonhosted.org/packages/25/0f/77bdef65a3b4a8160cb22b45ccb4e38be53c999f8ec791a1285251a5217d/snowflake_connector_python-5.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:615ec181d1b4e9980eb66adb0ce04ddb9e302a555e1452874b52c23afece4edd", size = 11650350, upload-time = "2026-10-15T11:03:55.304Z" }, + { url = "https://files.pythonhosted.org/packages/c9/84/0340c5f76fd9acac799f26a89af0e204f7ebd43a631ec3218927449b22af/snowflake_connector_python-5.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:83348aece70fd0ad0e4e60d71636188d69e80672fcb87d202fab83217a94e3b7", size = 11555944, upload-time = "2026-10-15T11:03:28.246Z" }, + { url = "https://files.pythonhosted.org/packages/61/27/0298144e727166edb67f13db14cb0a669e1ee413ae3eb50c44272c96f497/snowflake_connector_python-5.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:77ce8c006e0c765facb8b6fb05a20c87044602d0a16a7ec09c779f3809959183", size = 10886474, upload-time = "2026-10-15T11:03:19.399Z" }, + { url = "https://files.pythonhosted.org/packages/93/2c/6def82379a3d455b618415776015eb9ad67d391261b82c26650ba57d6725/snowflake_connector_python-5.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:3926b2df0f57ce67202fe7d5c399f026e806fe2f5431a5039215043d574de730", size = 17719188, upload-time = "2026-10-15T11:03:48.162Z" }, + { url = "https://files.pythonhosted.org/packages/dc/e9/d000c003a25108f14c4db8a446c2c44969692a91d3462cc9dc07f492086f/snowflake_connector_python-5.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:74eb680b69ef8f6990c42a66bd9883eb2e25207a5caef9e366ccf517e9259288", size = 18824177, upload-time = "2026-10-15T11:03:38.345Z" }, + { url = "https://files.pythonhosted.org/packages/71/f0/1d00669357595f207c7adafdfa2d41362cda519b43e150f65e757b4dde7c/snowflake_connector_python-5.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:d614ef97d88036c44ebf8c3a1ed5d1fa5131b64e9a4d24cdefb6e63ac7762239", size = 12201772, upload-time = "2026-10-15T11:03:57.396Z" }, +] + +[[package]] +name = "snowflake-sqlalchemy" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "snowflake-connector-python" }, + { name = "sqlalchemy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/b1/d829500045436ea5ce44e6bd7e92dad49206fb15425ffbcdcd8a25ddc098/snowflake_sqlalchemy-2.0.0.tar.gz", hash = "sha256:d80bfaf9ac7942aec46df951fb76552089114e7299227895746ea6da98830342", size = 356140, upload-time = "2026-10-19T08:28:55.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/6c/f455f5d4e12bba758fb8274c9ba1fdb507f7d7de1c96d4328e64d049c1cd/snowflake_sqlalchemy-2.0.0-py3-none-any.whl", hash = "sha256:177592383c8af1470ed05229a17507f6fcafb89d39e03df783ba12dc82d27bec", size = 163565, upload-time = "2026-10-19T08:28:54.133Z" }, +] + [[package]] name = "soupsieve" version = "2.9.2" @@ -4432,6 +4700,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0", size = 14154, upload-time = "2024-03-12T14:34:36.569Z" }, ] +[[package]] +name = "thrift" +version = "0.24.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/bd/8f90501b11206e545da3343ed0e5740fc694be24a5f637d7dd7e4e3af927/thrift-0.24.0.tar.gz", hash = "sha256:9ef601c49e988475ff0e741d8e1b45feec23b48514e524341efc274191f1789c", size = 69228, upload-time = "2026-07-11T16:53:50.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/3b/7ba9f07a7ac7b98dd7f29f2d9e6f6e98e332942dc8dbd82b71abbac1233d/thrift-0.24.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:873c93d8496cf71589efd3128ddc5af0be350e6c1a0e615475d840eaa54f6124", size = 228606, upload-time = "2026-07-11T16:53:14.788Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8c/4d0d03b40063504e75c9a74456dbca097ee78af92ddfae6c0216da9d2717/thrift-0.24.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:048d768e5822c436e8920b987d89a6a26e4d438ddf2de4b9d3f81444cd03cd04", size = 227999, upload-time = "2026-07-11T16:53:16.178Z" }, + { url = "https://files.pythonhosted.org/packages/12/43/74758d030d8d1f103f32026b92e0c9c8b14a6d190197a5e8055b9697294d/thrift-0.24.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2592bb0bf232d0808583626a7a8d71e265851e974c182967d67dde1f16902aee", size = 535383, upload-time = "2026-07-11T16:53:17.375Z" }, + { url = "https://files.pythonhosted.org/packages/89/d5/2f1e521dd0e8525c460e6e2ed3938f56b0d8208d794e08e0309f6e2ae299/thrift-0.24.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ca1b1ba00151565dc4e6673c924debd557aaf25b2790f1570b3430cb35503671", size = 540968, upload-time = "2026-07-11T16:53:18.61Z" }, + { url = "https://files.pythonhosted.org/packages/9b/02/44abc539a6356d8ffc47fd36724b376d4d4d904924a720b4416911cd8009/thrift-0.24.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:afb7977ccb8ecd7b1520a5141de64b58e94712528a26c10ac8f5b4ef220112a5", size = 1480581, upload-time = "2026-07-11T16:53:19.772Z" }, + { url = "https://files.pythonhosted.org/packages/34/bc/77a6c2f8f4983473a22905d7c420eb6d6ea46ee590bc287f3443533d0525/thrift-0.24.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c8d37a4311a87bda4bd2b8e4137eec638c18e3a57d11a1011bd98a8867523860", size = 1538742, upload-time = "2026-07-11T16:53:21.216Z" }, + { url = "https://files.pythonhosted.org/packages/06/b1/4c5e8ac71390292a70c4d75d695a562b6d6c081316d37f6121e18e1878d9/thrift-0.24.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbf461351940ddaa85bf8c2ee1754c9cfdd33bb78322e635e1ea5cd3947ae49a", size = 413572, upload-time = "2026-07-11T16:53:22.503Z" }, + { url = "https://files.pythonhosted.org/packages/0d/39/f3e6ec283fa6dbd490eab25a33c46edb6b3114ff9dc46b31bd88af3fd4ea/thrift-0.24.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:306e0fe3c96b300471c19cec60bd6816064fd93d04713488ef07120aff84653b", size = 223061, upload-time = "2026-07-11T16:53:23.804Z" }, + { url = "https://files.pythonhosted.org/packages/45/86/efd51714e5ce62c4d3c7f998eb1b1f5526cd1ee9fb4080545e7fc25fe5cf/thrift-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ac42199ea2a6fc6275b0aeec32ae09e7f9edb43890ff9a1af5e34191fb422cc", size = 222518, upload-time = "2026-07-11T16:53:24.885Z" }, + { url = "https://files.pythonhosted.org/packages/70/b0/4ea971750d449180363aa2273f1d80e66ce068643c5993589978d7a3951f/thrift-0.24.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cdd5f927eb98d0e8f00ab148b0cb66ae2598a396e907bd2b6febcbdcc46d80c", size = 531068, upload-time = "2026-07-11T16:53:26.044Z" }, + { url = "https://files.pythonhosted.org/packages/97/d8/cbd6352723ab4a6668c40c0d10a680b0f05178df2de9ea919ea8e8d2b834/thrift-0.24.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:134cd1a0d80f928377808348d1f9f647284ada093573beaa47040bed5507e219", size = 539609, upload-time = "2026-07-11T16:53:27.238Z" }, + { url = "https://files.pythonhosted.org/packages/46/b1/af2e258f9c135dc359acda9390e557ddc501a9185b65892fc253f94ca160/thrift-0.24.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:518199062feabfc0982767a0f7bceccb4fd1b485654f75e02913837444c3c130", size = 1474702, upload-time = "2026-07-11T16:53:28.669Z" }, + { url = "https://files.pythonhosted.org/packages/38/6f/93424dc6ceb2a561d055df4fee825ff7116e342e126c6aab357e0a94185e/thrift-0.24.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8c5de86af2a44641d423624c71c554a691d9f80c8b87709f15c7f98ccc6aeac", size = 1535581, upload-time = "2026-07-11T16:53:29.967Z" }, + { url = "https://files.pythonhosted.org/packages/c5/35/88debea28426b31ac417a83cda5c029f67cf72665fd2327562adb5c151b1/thrift-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:2f82bb16c4f2009dbc4fc9374604f05e998fb33be6a7787e095cea9842ecfa1d", size = 407669, upload-time = "2026-07-11T16:53:31.282Z" }, + { url = "https://files.pythonhosted.org/packages/93/5a/3b0d7b47ae73c28891997433447f87ba26de0fc11ca9fd4ea8c6b497bf23/thrift-0.24.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebb8b389142d67554d0de814dd6c1d62b962751f68721537efca429c57b09327", size = 224851, upload-time = "2026-07-11T16:53:32.528Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/4e17cce911ebbf76c59472b488dbe25c32d51cef2500b6e255a9fb59ff02/thrift-0.24.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18e7693f1bcef45937ad31a02564add55dec754710d42afc8ee3fc26ecf13028", size = 224313, upload-time = "2026-07-11T16:53:33.797Z" }, + { url = "https://files.pythonhosted.org/packages/fd/9a/eb8cb5a75fdff60b21b78c712a455adac01e1768792c3232184fa7202961/thrift-0.24.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937b398c311799fd5eddaeaffbd275510c68ead597eb1897e5e8113542fb2b50", size = 532517, upload-time = "2026-07-11T16:53:35.458Z" }, + { url = "https://files.pythonhosted.org/packages/30/b7/d0ecdf3573eb4026343d04efa64cf465a8453e699bd41a2d68156c3caf83/thrift-0.24.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b89a83d4e9ae6029e14f6f7c73305044bf02739d729c6aa8025ef3b6faef0ec0", size = 541124, upload-time = "2026-07-11T16:53:36.767Z" }, + { url = "https://files.pythonhosted.org/packages/df/ce/37daa851995ad441e38f83b2a8e67cdf68df10148779fb5bbafd4fb24f6b/thrift-0.24.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6381093f2c54e0f108554004a8cac3f1ef7ba99d6c5a9909c0eb64f450142685", size = 1476528, upload-time = "2026-07-11T16:53:38.056Z" }, + { url = "https://files.pythonhosted.org/packages/d6/67/80f3aeced7a38d17a281d1b9904ab9e93ab493f5c75e144b3180c278041c/thrift-0.24.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:28e034658d724aaa66007babb258ef12f0294036812fc449d7ce6fd15b07100f", size = 1537515, upload-time = "2026-07-11T16:53:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/37/1f/169f93d531a0d9546b6f18dacf37752c27cccfa719bf231162d1fcf8d0e0/thrift-0.24.0-cp313-cp313-win_amd64.whl", hash = "sha256:f4a44391ae1e32817553639b2991b0753d84487259fe87f8111c956c6bdebf43", size = 409450, upload-time = "2026-07-11T16:53:40.711Z" }, + { url = "https://files.pythonhosted.org/packages/01/46/e229eec0531070a76f2d98335f57ec29df5fb991d7a00e8b5aac2064e893/thrift-0.24.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7beb05268c76895f7c3130ce747a8a8cf35558fbc7f464f994e20a0c92181cbd", size = 227137, upload-time = "2026-07-11T16:53:41.893Z" }, + { url = "https://files.pythonhosted.org/packages/6e/30/27b0702a183de1fc884622a584093756eba6eb33a2a21760b279ff124573/thrift-0.24.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f40bdfbaf8d2795b1593bb75b4d9c77831bbe96a1ef59da4634ccc125a17dca5", size = 226617, upload-time = "2026-07-11T16:53:43.135Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b2/aa45a1d88c3692c14420095a3d3631c6add18e3c33527bdf5adfcaed5f1e/thrift-0.24.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:272624d36faa7bee01b94e5da5efc7305d9d3da57d005382c93fcbde8c3edf6a", size = 534129, upload-time = "2026-07-11T16:53:44.214Z" }, + { url = "https://files.pythonhosted.org/packages/7b/4f/b500ce0d9cf29a83cab88ab460ce7a7bab955f2a5d30bb1e7a9d4d4c2242/thrift-0.24.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bc91ae93005bc16c7362edcb58818a547850b9b52fe9b8075cc6b8922bcde2d", size = 543379, upload-time = "2026-07-11T16:53:45.394Z" }, + { url = "https://files.pythonhosted.org/packages/32/77/0ecb68cf0ea0af2c51ba88cf7f85ce59823efc7a7cb7573abaac0d60d49b/thrift-0.24.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c05bb4eac921836c12cdf30c237283df8a42aff6540f7970cd72c5959b139970", size = 1478854, upload-time = "2026-07-11T16:53:46.654Z" }, + { url = "https://files.pythonhosted.org/packages/fa/89/fd2a0b7ef3a54bac545fdaf9e34b3d3012471058a9eddbe5a2697bf69773/thrift-0.24.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e5da6b391828d46df9471f22181374760a91773b7e07ad7eb44f351c90580662", size = 1539741, upload-time = "2026-07-11T16:53:48.012Z" }, + { url = "https://files.pythonhosted.org/packages/a5/10/7b3368202c2333d448fa9b13288ecb8f68e9361531c8757a7b1eb27ab750/thrift-0.24.0-cp314-cp314-win_amd64.whl", hash = "sha256:829db909a053d4064cb9fe574cc1f5994c24d727f61cdf6446ca774cd3ba5268", size = 419738, upload-time = "2026-07-11T16:53:49.773Z" }, +] + [[package]] name = "tinycss2" version = "1.5.1" @@ -4444,6 +4748,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/45/c7b5c3168458db837e8ceab06dc77824e18202679d0463f0e8f002143a97/tinycss2-1.5.1-py3-none-any.whl", hash = "sha256:3415ba0f5839c062696996998176c4a3751d18b7edaaeeb658c9ce21ec150661", size = 28404, upload-time = "2025-11-23T10:29:08.676Z" }, ] +[[package]] +name = "tomlkit" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, +] + [[package]] name = "tornado" version = "6.5.8" diff --git a/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json b/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json new file mode 100644 index 000000000..8540b5396 --- /dev/null +++ b/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json @@ -0,0 +1,69 @@ +{ + "note": "The environment both suites read to reach a real warehouse. Held here so the two harnesses cannot drift on a variable name. Connection variables are the vendor's own, which is what the Databricks and Snowflake client libraries read in Python and what odbc::databricks() and odbc::snowflake() read in R, so one export block serves both languages and the vendor CLIs. Fixture variables carry one dotted identifier each. A backend whose required variables are unset skips; an optional fixture skips only the tests that need it.", + "backends": { + "databricks": { + "connection": { + "required": ["DATABRICKS_HOST", "DATABRICKS_HTTP_PATH"], + "token": "DATABRICKS_TOKEN" + }, + "fixtures": { + "schema": { + "variable": "COMMONS_TEST_DATABRICKS_SCHEMA", + "required": true, + "arity": 2, + "description": "Catalog and schema holding the provisioned tables, as `catalog.schema`. The table names themselves come from live-warehouse-rows.json." + }, + "denied_table": { + "variable": "COMMONS_TEST_DATABRICKS_DENIED_TABLE", + "required": false, + "arity": 3, + "description": "A relation the test principal genuinely cannot read. Needs an owner other than the test principal, so provisioning cannot finish it." + }, + "metric_view": { + "variable": "COMMONS_TEST_DATABRICKS_METRIC_VIEW", + "required": false, + "arity": 3, + "description": "A parameterized metric view. Read by the R suite only: pkg-py has no semantic-model surface (commons kata gcgj)." + } + } + }, + "snowflake": { + "connection": { + "required": ["SNOWFLAKE_ACCOUNT", "SNOWFLAKE_USER", "SNOWFLAKE_WAREHOUSE"], + "token": "SNOWFLAKE_PASSWORD" + }, + "fixtures": { + "schema": { + "variable": "COMMONS_TEST_SNOWFLAKE_SCHEMA", + "required": true, + "arity": 2, + "description": "Database and schema holding the provisioned tables, as `database.schema`." + }, + "denied_table": { + "variable": "COMMONS_TEST_SNOWFLAKE_DENIED_TABLE", + "required": false, + "arity": 3, + "description": "A relation the test role genuinely cannot read." + }, + "alternate_role": { + "variable": "COMMONS_TEST_SNOWFLAKE_ALTERNATE_ROLE", + "required": false, + "arity": 1, + "description": "A second usable role, for the active-role change test." + }, + "semantic_view": { + "variable": "COMMONS_TEST_SNOWFLAKE_SEMANTIC_VIEW", + "required": false, + "arity": 3, + "description": "A semantic view. Read by the R suite only, as with the Databricks metric view." + }, + "verified_query_view": { + "variable": "COMMONS_TEST_SNOWFLAKE_VERIFIED_QUERY_VIEW", + "required": false, + "arity": 3, + "description": "A semantic view carrying a verified query. Read by the R suite only." + } + } + } + } +} diff --git a/pkg-r/tests/testthat/fixtures/shared/live-warehouse-rows.json b/pkg-r/tests/testthat/fixtures/shared/live-warehouse-rows.json new file mode 100644 index 000000000..3998f2324 --- /dev/null +++ b/pkg-r/tests/testthat/fixtures/shared/live-warehouse-rows.json @@ -0,0 +1,389 @@ +{ + "note": "The tables a live warehouse run is provisioned with, and the rows in them. The three tables and their columns are the ones the corpus dictionaries in definition-export/valid/ describe, so the SQL pinned in definition-warehouse-sql.json can be executed against real relations instead of only being agreed on. Both suites load these rows into DuckDB as well and compare the two engines' answers, so the rows cannot live in the provisioning SQL alone. Values are chosen to make the definitions non-trivial: a value that is null, out of range, oddly cased, or sitting just under a threshold is there because a definition asks about it. Special doubles travel as the strings `NaN`, `Infinity` and `-Infinity`, since JSON has no spelling for them.", + "types": { + "int64": { + "duckdb": "BIGINT", + "databricks": "BIGINT", + "snowflake": "NUMBER(38,0)" + }, + "double": { + "duckdb": "DOUBLE", + "databricks": "DOUBLE", + "snowflake": "FLOAT" + }, + "string": { + "duckdb": "VARCHAR", + "databricks": "STRING", + "snowflake": "VARCHAR" + }, + "boolean": { + "duckdb": "BOOLEAN", + "databricks": "BOOLEAN", + "snowflake": "BOOLEAN" + }, + "date": { + "duckdb": "DATE", + "databricks": "DATE", + "snowflake": "DATE" + }, + "timestamp": { + "duckdb": "TIMESTAMPTZ", + "databricks": "TIMESTAMP", + "snowflake": "TIMESTAMP_TZ" + }, + "string_array": { + "duckdb": "VARCHAR[]", + "databricks": "ARRAY", + "snowflake": "ARRAY" + }, + "profile_struct": { + "duckdb": "STRUCT(zip VARCHAR, geo STRUCT(latitude DOUBLE), \"nick names\" VARCHAR)", + "databricks": "STRUCT, `nick names`: STRING>", + "snowflake": "OBJECT(zip VARCHAR, geo OBJECT(latitude FLOAT), \"nick names\" VARCHAR)" + } + }, + "tables": { + "orders": { + "dictionary": "core.yaml", + "columns": [ + { + "name": "status_cd", + "type": "int64" + }, + { + "name": "order_total", + "type": "double" + }, + { + "name": "tile_size", + "type": "string" + } + ], + "rows": [ + { + "status_cd": 10, + "order_total": 100.5, + "tile_size": "Consumer" + }, + { + "status_cd": 90, + "order_total": 250.0, + "tile_size": "Enterprise-1" + }, + { + "status_cd": 10, + "order_total": 250.0, + "tile_size": "Enterprise-1" + }, + { + "status_cd": 10, + "order_total": 75.25, + "tile_size": "Mid-Market-3" + }, + { + "status_cd": 90, + "order_total": 10.0, + "tile_size": "Consumer" + }, + { + "status_cd": 10, + "order_total": null, + "tile_size": "Mid-Market-3" + }, + { + "status_cd": 10, + "order_total": 0.0, + "tile_size": "enterprise-1" + } + ], + "why": { + "status_cd 90": "net_revenue zeroes a returned order, so it must differ from a plain SUM.", + "order_total null": "SUM and AVG over a null, and list_price of a null.", + "tile_size enterprise-1": "Lower cased, so is_enterprise's IN list is tested for case sensitivity rather than assumed.", + "order_total 250.0 twice": "COUNT_DISTINCT must differ from COUNT." + } + }, + "values": { + "dictionary": "functions.yaml", + "columns": [ + { + "name": "number", + "type": "double" + }, + { + "name": "text", + "type": "string" + }, + { + "name": "flag", + "type": "boolean" + } + ], + "rows": [ + { + "number": 4.567, + "text": "Apple", + "flag": true + }, + { + "number": -5.0, + "text": "abZ", + "flag": false + }, + { + "number": 7.0, + "text": "A-1", + "flag": true + }, + { + "number": 0.0, + "text": "\tapricot ", + "flag": true + }, + { + "number": "NaN", + "text": "banana", + "flag": false + }, + { + "number": "Infinity", + "text": "AZ", + "flag": true + }, + { + "number": "-Infinity", + "text": null, + "flag": false + }, + { + "number": 2.5, + "text": "A_x", + "flag": true + }, + { + "number": 4.567, + "text": "a\\z", + "flag": null + } + ], + "why": { + "number NaN and Infinity": "finite calls IS_NAN, IS_INFINITE and IS_FINITE, which is also why the column is a double rather than a decimal.", + "number -5.0 and 7.0": "MOD over a negative operand, where a floored modulus and a truncated one disagree.", + "text tab and trailing space": "definition-warehouse-sql.json records that Databricks trim removes spaces where data-dict removes whitespace. Only a tab makes that divergence observable.", + "text backslash": "The same file records that a Databricks LIKE pattern treats backslash as an escape.", + "text A_x": "The literal LIKE 'A_%' pattern, where the underscore is a wildcard.", + "text null and flag null": "COUNT skips nulls, and ANY and ALL fold over one.", + "number 4.567 twice": "COUNT_DISTINCT must differ from COUNT." + } + }, + "survey": { + "dictionary": "language.yaml", + "columns": [ + { + "name": "q1", + "type": "boolean" + }, + { + "name": "q2", + "type": "boolean" + }, + { + "name": "created", + "type": "date" + }, + { + "name": "observed", + "type": "timestamp" + }, + { + "name": "amount", + "type": "double" + }, + { + "name": "ratio", + "type": "double" + }, + { + "name": "pattern", + "type": "string" + }, + { + "name": "category", + "type": "string" + }, + { + "name": "profile", + "type": "profile_struct" + }, + { + "name": "tags", + "type": "string_array" + }, + { + "name": "untyped", + "type": "string" + } + ], + "rows": [ + { + "q1": true, + "q2": true, + "created": "2024-06-15", + "observed": "2020-03-01T12:00:00Z", + "amount": 3.25, + "ratio": 0.5, + "pattern": "a", + "category": "A", + "profile": { + "zip": "12345", + "geo": { + "latitude": 38.9 + }, + "nick names": "Ada" + }, + "tags": [ + "north", + "west" + ], + "untyped": "one" + }, + { + "q1": false, + "q2": true, + "created": "2019-12-31", + "observed": "2030-07-04T00:00:00Z", + "amount": -7.5, + "ratio": 2.0, + "pattern": "abc", + "category": "B", + "profile": { + "zip": "123456", + "geo": { + "latitude": -12.25 + }, + "nick names": "Bo" + }, + "tags": [ + "south" + ], + "untyped": null + }, + { + "q1": null, + "q2": false, + "created": "2024-01-01", + "observed": "2024-01-01T01:30:00-06:00", + "amount": 0.1234567890123456, + "ratio": -1.0, + "pattern": "9%", + "category": "A", + "profile": { + "zip": "9%", + "geo": { + "latitude": 0.0 + }, + "nick names": "Cy Q" + }, + "tags": [], + "untyped": "three" + }, + { + "q1": true, + "q2": null, + "created": "2023-12-31", + "observed": "2024-01-01T01:30:00.123Z", + "amount": 0.2, + "ratio": 0.0, + "pattern": "xyz", + "category": "B", + "profile": { + "zip": "007", + "geo": { + "latitude": 89.5 + }, + "nick names": "Dee" + }, + "tags": null, + "untyped": "four" + }, + { + "q1": false, + "q2": false, + "created": "2020-01-01", + "observed": "2019-06-30T23:59:59Z", + "amount": 0.0, + "ratio": -0.5, + "pattern": "az", + "category": "A", + "profile": { + "zip": "", + "geo": { + "latitude": null + }, + "nick names": "" + }, + "tags": [ + "east", + "east" + ], + "untyped": null + }, + { + "q1": true, + "q2": false, + "created": null, + "observed": null, + "amount": 1.0, + "ratio": 1.5, + "pattern": null, + "category": "B", + "profile": null, + "tags": [ + "north" + ], + "untyped": "six" + }, + { + "q1": true, + "q2": true, + "created": "2026-02-28", + "observed": "2031-12-31T18:45:00Z", + "amount": 12.0, + "ratio": 0.25, + "pattern": "a%b", + "category": "A", + "profile": { + "zip": "54321", + "geo": { + "latitude": 45.0 + }, + "nick names": "Gus" + }, + "tags": [ + "west", + "north" + ], + "untyped": "seven" + } + ], + "why": { + "observed far from now": "fresh is `observed >= NOW() - interval(2, weeks)`, so its answer depends on when the test runs. Every value sits years either side of that boundary, which keeps the answer stable without recomputing anything at test time.", + "observed offsets": "offset time and fractional time compare against an offset literal and a fractional-second literal, so a row carries each.", + "created either side of 2024-01-01": "recent keys off that date, and selected dates off 2020-01-01.", + "created null": "nullable date shift adds NULL to a date, and mixed temporal case falls through to observed.", + "q1 and q2 nulls": "complete asks whether both are non-null, case without else returns null for a null condition, and null logic folds one.", + "amount 0.1234567890123456": "precise threshold compares against 0.12345678901234567, so a value just below it is the only one that tests the comparison.", + "amount -7.5 and 0.0 and 1.0": "range predicate is BETWEEN -10 AND 10 with NOT IN (0, 1), and amount band is relative to MIN(amount).", + "ratio -1.0": "negative quotient divides by ratio + 1.0, so this row divides by zero. DuckDB and Databricks are expected to disagree here, and that disagreement is a finding to record rather than a value to pick around.", + "profile zip 9%": "dynamic match is `pattern LIKE profile.zip`, so the pattern comes from data and one row's zip has to contain a wildcard.", + "profile null and latitude null": "deep score reaches through two levels of struct, and postal length takes the length of a field that may be absent.", + "profile nick names with a space": "The field name carries a space, which is what makes quoting in the emitted SQL observable.", + "tags empty and null": "missing tags asks only about null, so an empty array must not answer the same way." + } + } + }, + "unrunnable": { + "note": "Definitions the corpus declares that cannot be executed against these rows on any engine, so neither the live comparison nor the DuckDB run attempts them. A key here is a finding, not a convenience: it records a definition whose emitted SQL does not run over a plausible column type.", + "survey::computed interval": "data-dict lowers `interval(amount + 1, days)` to `observed + (amount + 1 * INTERVAL '1 days')`, which DuckDB refuses to bind because amount is a double and there is no double-times-interval operator. amount has to be fractional for `precise threshold` to compare anything, so no single column type satisfies both, and this is the one that gives way. Upstream's own translation is what fails here, which makes it a data-dict question rather than a commons emitter one; raise it with commons kata g0ax." + } +} diff --git a/scripts/provision-live-warehouse.py b/scripts/provision-live-warehouse.py new file mode 100755 index 000000000..47e29b404 --- /dev/null +++ b/scripts/provision-live-warehouse.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Emit the SQL that brings a warehouse to the state the live tests expect. + +The tables, columns and rows come from ``tests/shared/live-warehouse-rows.json`` +so that the warehouse and the DuckDB the suites compare against hold the same +data. Nothing is generated into the repository: the SQL is printed for a person +to run, which leaves no committed artifact to go stale. + + python3 scripts/provision-live-warehouse.py databricks main.commons_live + +The denied-table fixture is not created here. A relation its owner cannot read +does not exist, so making one needs a second principal; the script prints what +remains to be done by hand. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +FIXTURE = Path(__file__).resolve().parent.parent / "tests" / "shared" / "live-warehouse-rows.json" + +SPECIAL_DOUBLES = { + "databricks": {"NaN": "double('NaN')", "Infinity": "double('Infinity')", "-Infinity": "double('-Infinity')"}, + "snowflake": {"NaN": "'NaN'::FLOAT", "Infinity": "'Inf'::FLOAT", "-Infinity": "'-Inf'::FLOAT"}, +} + + +def quote_string(value: str) -> str: + """A string literal for either warehouse. + + Both treat a backslash inside a literal as an escape, so both need the + same doubling, and a tab is written as an escape rather than embedded so + the generated SQL survives being pasted into a query editor. + """ + escaped = value.replace("\\", "\\\\").replace("'", "''").replace("\t", "\\t").replace("\n", "\\n") + return f"'{escaped}'" + + +def literal(value: Any, type_name: str, types: dict[str, Any], dialect: str) -> str: + if value is None: + return f"CAST(NULL AS {types[type_name][dialect]})" + if type_name == "boolean": + return "TRUE" if value else "FALSE" + if type_name == "int64": + return str(value) + if type_name == "double": + if isinstance(value, str): + return SPECIAL_DOUBLES[dialect][value] + return repr(float(value)) + if type_name == "string": + return quote_string(value) + if type_name == "date": + return f"DATE {quote_string(value)}" + if type_name == "timestamp": + stamp = value.replace("T", " ").replace("Z", "+00:00") + if dialect == "databricks": + return f"TIMESTAMP {quote_string(stamp)}" + return f"{quote_string(stamp)}::TIMESTAMP_TZ" + if type_name == "string_array": + items = ", ".join(quote_string(item) for item in value) + if dialect == "databricks": + return f"CAST(array({items}) AS ARRAY)" + return f"ARRAY_CONSTRUCT({items})" + if type_name == "profile_struct": + return struct_literal(value, types[type_name][dialect], dialect) + raise ValueError(f"No literal rule for type {type_name!r}") + + +def struct_literal(value: dict[str, Any], declared: str, dialect: str) -> str: + """The profile column, whose shape is fixed by the corpus dictionary. + + Written against that one shape rather than driven by the declared type, + because a general struct renderer would have to parse two dialects' type + grammars to learn the field types, and there is only one struct here. + """ + latitude = value["geo"]["latitude"] if value.get("geo") else None + if dialect == "databricks": + geo = "CAST(NULL AS DOUBLE)" if latitude is None else repr(float(latitude)) + inner = f"named_struct('latitude', {geo})" + fields = ( + f"'zip', {quote_string(value['zip'])}, " + f"'geo', {inner}, " + f"'nick names', {quote_string(value['nick names'])}" + ) + return f"CAST(named_struct({fields}) AS {declared})" + geo = "NULL" if latitude is None else repr(float(latitude)) + inner = f"OBJECT_CONSTRUCT_KEEP_NULL('latitude', {geo})" + fields = ( + f"'zip', {quote_string(value['zip'])}, " + f"'geo', {inner}, " + f"'nick names', {quote_string(value['nick names'])}" + ) + return f"CAST(OBJECT_CONSTRUCT_KEEP_NULL({fields}) AS {declared})" + + +def quote_identifier(name: str, dialect: str) -> str: + if dialect == "databricks": + return "`" + name.replace("`", "``") + "`" + return '"' + name.replace('"', '""') + '"' + + +def render(dialect: str, namespace: str) -> str: + fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + types = fixture["types"] + parts = namespace.split(".") + if len(parts) != 2: + raise SystemExit(f"Expected `catalog.schema`, got {namespace!r}") + qualified = ".".join(quote_identifier(part, dialect) for part in parts) + + lines = [ + f"-- Generated by scripts/provision-live-warehouse.py for {dialect}.", + "-- Rows come from tests/shared/live-warehouse-rows.json; edit that file, not this output.", + "", + f"CREATE SCHEMA IF NOT EXISTS {qualified};", + "", + ] + + for table, spec in fixture["tables"].items(): + target = f"{qualified}.{quote_identifier(table, dialect)}" + columns = ",\n".join( + f" {quote_identifier(column['name'], dialect)} {types[column['type']][dialect]}" + for column in spec["columns"] + ) + lines += [ + f"DROP TABLE IF EXISTS {target};", + f"CREATE TABLE {target} (", + columns, + ");", + "", + ] + column_list = ", ".join(quote_identifier(c["name"], dialect) for c in spec["columns"]) + values = ",\n".join( + " (" + + ", ".join( + literal(row[column["name"]], column["type"], types, dialect) + for column in spec["columns"] + ) + + ")" + for row in spec["rows"] + ) + lines += [f"INSERT INTO {target} ({column_list}) VALUES", values + ";", ""] + + lines += [ + "-- Still to do by hand, because an owner can always read their own relation:", + f"-- Create a copy of {qualified}.{quote_identifier('orders', dialect)} owned by another", + "-- principal, in a schema this one has no SELECT on, and export its identifier as the", + "-- denied-table variable named in tests/shared/live-warehouse-env.json. Until then the", + "-- access-refusal tests skip.", + ] + return "\n".join(lines) + "\n" + + +def main(argv: list[str]) -> int: + if len(argv) != 3 or argv[1] not in ("databricks", "snowflake"): + print(__doc__, file=sys.stderr) + print("Usage: provision-live-warehouse.py {databricks|snowflake} ", file=sys.stderr) + return 2 + print(render(argv[1], argv[2]), end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/tests/shared/README.md b/tests/shared/README.md index 37b5a6a02..7a37f4d66 100644 --- a/tests/shared/README.md +++ b/tests/shared/README.md @@ -51,6 +51,8 @@ The Python suite reads this directory directly. The R suite cannot. `testthat` n - **The execution tool's output.** `execution-repl.json` pins what the model reads back from `run_r` and `run_python` for code that runs to completion: printed text, a trailing value, nothing for an assignment or an invisible or `None` value, stderr, the order of text and value, variables that persist across calls, and the text kept before an error. Each case gives the code in both languages and runs in a real worker. Values print each language's own way, and R keeps the newline that ends printed text when it joins the next output, so cases that show either give per-language text. Three differences are left out on purpose and listed in the fixture: R shows every visible top-level value and Python only a trailing one, R puts two writes to one line on separate lines, and an error is R's condition message but Python's traceback. - **The worker lifecycle.** `worker-lifecycle.json` pins the worker's time limits (call, interrupt grace, idle) and the tool result the model reads when a call is interrupted, when the worker does not answer the interrupt, and when the worker crashes. The messages are templates with `{language}` and `{seconds}` placeholders, and each case gives the code that causes the failure in each language. Failures only one driver can detect, such as a call the user stopped in Python, are left out. - **Plots and styled tables from measures.** `measure-rich-results.json` pins what the model and the user get when a measure returns a plot (R's `ggplot`; Python's matplotlib figure) or a styled table (R's `gt`; Python's `great_tables`): the notes the model reads, the plot's alt text, the note when the plot or table cannot be shown, and that a table's handle is its data frame. Each case gives the code for each language and fails for real where it pins a failure. A plot's handle is left out, since R keeps the plot object and Python keeps none. +- **The live warehouse environment.** `live-warehouse-env.json` names the environment variables both suites read to reach a real Snowflake or Databricks: the vendor's own connection variables, which the Python client libraries and `odbc::databricks()` already read, and a `COMMONS_TEST_*` variable per fixture carrying one dotted identifier. It is held here because the two harnesses must agree on a name, and a name spelled in two places drifts. It is the one fixture here that pins harness configuration rather than observable behavior. +- **The live warehouse rows.** `live-warehouse-rows.json` holds the three tables a live run is provisioned with, their column types per dialect, and their rows. The tables and columns are the ones the corpus dictionaries in `definition-export/valid/` describe, which is what lets the SQL pinned in `definition-warehouse-sql.json` be executed rather than only agreed on. Both suites also load these rows into DuckDB and compare the two engines' answers, so the rows cannot live in the provisioning SQL alone. Every value is there because a definition asks about it, and each table's `why` says which. Its `unrunnable` section records definitions whose emitted SQL does not run over a plausible column type, which is a finding rather than a convenience. Generate the provisioning SQL with `scripts/provision-live-warehouse.py`. - **Trace file naming.** `trace(-[0-9]+)?\.jsonl`, one OTLP envelope per line. - **The AF_UNIX boundary.** `sandbox-abstract-ipc.json` pins which address-taking socket calls the seccomp network block screens under `network="none"` — `always` for the ones denied outright, `when_addressed` for `sendto`, which stays open with a NULL address because libc spells `send()` that way — the `socketcall` sub-calls that stay allowed on i386, where glibc routes `socketpair()`, `send()`, and `recv()` through `socketcall`, and the Landlock ABI from which the ruleset scopes abstract sockets. It also pins the Landlock ABI from which the ruleset handles the right to reach pathname sockets, and that read roots never get that right. The Python runner checks its filter tables and constants against the fixture; the R runner checks the same names in `sandbox.c`, since an installed seccomp filter cannot be introspected. Hand-maintained. diff --git a/tests/shared/live-warehouse-env.json b/tests/shared/live-warehouse-env.json new file mode 100644 index 000000000..8540b5396 --- /dev/null +++ b/tests/shared/live-warehouse-env.json @@ -0,0 +1,69 @@ +{ + "note": "The environment both suites read to reach a real warehouse. Held here so the two harnesses cannot drift on a variable name. Connection variables are the vendor's own, which is what the Databricks and Snowflake client libraries read in Python and what odbc::databricks() and odbc::snowflake() read in R, so one export block serves both languages and the vendor CLIs. Fixture variables carry one dotted identifier each. A backend whose required variables are unset skips; an optional fixture skips only the tests that need it.", + "backends": { + "databricks": { + "connection": { + "required": ["DATABRICKS_HOST", "DATABRICKS_HTTP_PATH"], + "token": "DATABRICKS_TOKEN" + }, + "fixtures": { + "schema": { + "variable": "COMMONS_TEST_DATABRICKS_SCHEMA", + "required": true, + "arity": 2, + "description": "Catalog and schema holding the provisioned tables, as `catalog.schema`. The table names themselves come from live-warehouse-rows.json." + }, + "denied_table": { + "variable": "COMMONS_TEST_DATABRICKS_DENIED_TABLE", + "required": false, + "arity": 3, + "description": "A relation the test principal genuinely cannot read. Needs an owner other than the test principal, so provisioning cannot finish it." + }, + "metric_view": { + "variable": "COMMONS_TEST_DATABRICKS_METRIC_VIEW", + "required": false, + "arity": 3, + "description": "A parameterized metric view. Read by the R suite only: pkg-py has no semantic-model surface (commons kata gcgj)." + } + } + }, + "snowflake": { + "connection": { + "required": ["SNOWFLAKE_ACCOUNT", "SNOWFLAKE_USER", "SNOWFLAKE_WAREHOUSE"], + "token": "SNOWFLAKE_PASSWORD" + }, + "fixtures": { + "schema": { + "variable": "COMMONS_TEST_SNOWFLAKE_SCHEMA", + "required": true, + "arity": 2, + "description": "Database and schema holding the provisioned tables, as `database.schema`." + }, + "denied_table": { + "variable": "COMMONS_TEST_SNOWFLAKE_DENIED_TABLE", + "required": false, + "arity": 3, + "description": "A relation the test role genuinely cannot read." + }, + "alternate_role": { + "variable": "COMMONS_TEST_SNOWFLAKE_ALTERNATE_ROLE", + "required": false, + "arity": 1, + "description": "A second usable role, for the active-role change test." + }, + "semantic_view": { + "variable": "COMMONS_TEST_SNOWFLAKE_SEMANTIC_VIEW", + "required": false, + "arity": 3, + "description": "A semantic view. Read by the R suite only, as with the Databricks metric view." + }, + "verified_query_view": { + "variable": "COMMONS_TEST_SNOWFLAKE_VERIFIED_QUERY_VIEW", + "required": false, + "arity": 3, + "description": "A semantic view carrying a verified query. Read by the R suite only." + } + } + } + } +} diff --git a/tests/shared/live-warehouse-rows.json b/tests/shared/live-warehouse-rows.json new file mode 100644 index 000000000..3998f2324 --- /dev/null +++ b/tests/shared/live-warehouse-rows.json @@ -0,0 +1,389 @@ +{ + "note": "The tables a live warehouse run is provisioned with, and the rows in them. The three tables and their columns are the ones the corpus dictionaries in definition-export/valid/ describe, so the SQL pinned in definition-warehouse-sql.json can be executed against real relations instead of only being agreed on. Both suites load these rows into DuckDB as well and compare the two engines' answers, so the rows cannot live in the provisioning SQL alone. Values are chosen to make the definitions non-trivial: a value that is null, out of range, oddly cased, or sitting just under a threshold is there because a definition asks about it. Special doubles travel as the strings `NaN`, `Infinity` and `-Infinity`, since JSON has no spelling for them.", + "types": { + "int64": { + "duckdb": "BIGINT", + "databricks": "BIGINT", + "snowflake": "NUMBER(38,0)" + }, + "double": { + "duckdb": "DOUBLE", + "databricks": "DOUBLE", + "snowflake": "FLOAT" + }, + "string": { + "duckdb": "VARCHAR", + "databricks": "STRING", + "snowflake": "VARCHAR" + }, + "boolean": { + "duckdb": "BOOLEAN", + "databricks": "BOOLEAN", + "snowflake": "BOOLEAN" + }, + "date": { + "duckdb": "DATE", + "databricks": "DATE", + "snowflake": "DATE" + }, + "timestamp": { + "duckdb": "TIMESTAMPTZ", + "databricks": "TIMESTAMP", + "snowflake": "TIMESTAMP_TZ" + }, + "string_array": { + "duckdb": "VARCHAR[]", + "databricks": "ARRAY", + "snowflake": "ARRAY" + }, + "profile_struct": { + "duckdb": "STRUCT(zip VARCHAR, geo STRUCT(latitude DOUBLE), \"nick names\" VARCHAR)", + "databricks": "STRUCT, `nick names`: STRING>", + "snowflake": "OBJECT(zip VARCHAR, geo OBJECT(latitude FLOAT), \"nick names\" VARCHAR)" + } + }, + "tables": { + "orders": { + "dictionary": "core.yaml", + "columns": [ + { + "name": "status_cd", + "type": "int64" + }, + { + "name": "order_total", + "type": "double" + }, + { + "name": "tile_size", + "type": "string" + } + ], + "rows": [ + { + "status_cd": 10, + "order_total": 100.5, + "tile_size": "Consumer" + }, + { + "status_cd": 90, + "order_total": 250.0, + "tile_size": "Enterprise-1" + }, + { + "status_cd": 10, + "order_total": 250.0, + "tile_size": "Enterprise-1" + }, + { + "status_cd": 10, + "order_total": 75.25, + "tile_size": "Mid-Market-3" + }, + { + "status_cd": 90, + "order_total": 10.0, + "tile_size": "Consumer" + }, + { + "status_cd": 10, + "order_total": null, + "tile_size": "Mid-Market-3" + }, + { + "status_cd": 10, + "order_total": 0.0, + "tile_size": "enterprise-1" + } + ], + "why": { + "status_cd 90": "net_revenue zeroes a returned order, so it must differ from a plain SUM.", + "order_total null": "SUM and AVG over a null, and list_price of a null.", + "tile_size enterprise-1": "Lower cased, so is_enterprise's IN list is tested for case sensitivity rather than assumed.", + "order_total 250.0 twice": "COUNT_DISTINCT must differ from COUNT." + } + }, + "values": { + "dictionary": "functions.yaml", + "columns": [ + { + "name": "number", + "type": "double" + }, + { + "name": "text", + "type": "string" + }, + { + "name": "flag", + "type": "boolean" + } + ], + "rows": [ + { + "number": 4.567, + "text": "Apple", + "flag": true + }, + { + "number": -5.0, + "text": "abZ", + "flag": false + }, + { + "number": 7.0, + "text": "A-1", + "flag": true + }, + { + "number": 0.0, + "text": "\tapricot ", + "flag": true + }, + { + "number": "NaN", + "text": "banana", + "flag": false + }, + { + "number": "Infinity", + "text": "AZ", + "flag": true + }, + { + "number": "-Infinity", + "text": null, + "flag": false + }, + { + "number": 2.5, + "text": "A_x", + "flag": true + }, + { + "number": 4.567, + "text": "a\\z", + "flag": null + } + ], + "why": { + "number NaN and Infinity": "finite calls IS_NAN, IS_INFINITE and IS_FINITE, which is also why the column is a double rather than a decimal.", + "number -5.0 and 7.0": "MOD over a negative operand, where a floored modulus and a truncated one disagree.", + "text tab and trailing space": "definition-warehouse-sql.json records that Databricks trim removes spaces where data-dict removes whitespace. Only a tab makes that divergence observable.", + "text backslash": "The same file records that a Databricks LIKE pattern treats backslash as an escape.", + "text A_x": "The literal LIKE 'A_%' pattern, where the underscore is a wildcard.", + "text null and flag null": "COUNT skips nulls, and ANY and ALL fold over one.", + "number 4.567 twice": "COUNT_DISTINCT must differ from COUNT." + } + }, + "survey": { + "dictionary": "language.yaml", + "columns": [ + { + "name": "q1", + "type": "boolean" + }, + { + "name": "q2", + "type": "boolean" + }, + { + "name": "created", + "type": "date" + }, + { + "name": "observed", + "type": "timestamp" + }, + { + "name": "amount", + "type": "double" + }, + { + "name": "ratio", + "type": "double" + }, + { + "name": "pattern", + "type": "string" + }, + { + "name": "category", + "type": "string" + }, + { + "name": "profile", + "type": "profile_struct" + }, + { + "name": "tags", + "type": "string_array" + }, + { + "name": "untyped", + "type": "string" + } + ], + "rows": [ + { + "q1": true, + "q2": true, + "created": "2024-06-15", + "observed": "2020-03-01T12:00:00Z", + "amount": 3.25, + "ratio": 0.5, + "pattern": "a", + "category": "A", + "profile": { + "zip": "12345", + "geo": { + "latitude": 38.9 + }, + "nick names": "Ada" + }, + "tags": [ + "north", + "west" + ], + "untyped": "one" + }, + { + "q1": false, + "q2": true, + "created": "2019-12-31", + "observed": "2030-07-04T00:00:00Z", + "amount": -7.5, + "ratio": 2.0, + "pattern": "abc", + "category": "B", + "profile": { + "zip": "123456", + "geo": { + "latitude": -12.25 + }, + "nick names": "Bo" + }, + "tags": [ + "south" + ], + "untyped": null + }, + { + "q1": null, + "q2": false, + "created": "2024-01-01", + "observed": "2024-01-01T01:30:00-06:00", + "amount": 0.1234567890123456, + "ratio": -1.0, + "pattern": "9%", + "category": "A", + "profile": { + "zip": "9%", + "geo": { + "latitude": 0.0 + }, + "nick names": "Cy Q" + }, + "tags": [], + "untyped": "three" + }, + { + "q1": true, + "q2": null, + "created": "2023-12-31", + "observed": "2024-01-01T01:30:00.123Z", + "amount": 0.2, + "ratio": 0.0, + "pattern": "xyz", + "category": "B", + "profile": { + "zip": "007", + "geo": { + "latitude": 89.5 + }, + "nick names": "Dee" + }, + "tags": null, + "untyped": "four" + }, + { + "q1": false, + "q2": false, + "created": "2020-01-01", + "observed": "2019-06-30T23:59:59Z", + "amount": 0.0, + "ratio": -0.5, + "pattern": "az", + "category": "A", + "profile": { + "zip": "", + "geo": { + "latitude": null + }, + "nick names": "" + }, + "tags": [ + "east", + "east" + ], + "untyped": null + }, + { + "q1": true, + "q2": false, + "created": null, + "observed": null, + "amount": 1.0, + "ratio": 1.5, + "pattern": null, + "category": "B", + "profile": null, + "tags": [ + "north" + ], + "untyped": "six" + }, + { + "q1": true, + "q2": true, + "created": "2026-02-28", + "observed": "2031-12-31T18:45:00Z", + "amount": 12.0, + "ratio": 0.25, + "pattern": "a%b", + "category": "A", + "profile": { + "zip": "54321", + "geo": { + "latitude": 45.0 + }, + "nick names": "Gus" + }, + "tags": [ + "west", + "north" + ], + "untyped": "seven" + } + ], + "why": { + "observed far from now": "fresh is `observed >= NOW() - interval(2, weeks)`, so its answer depends on when the test runs. Every value sits years either side of that boundary, which keeps the answer stable without recomputing anything at test time.", + "observed offsets": "offset time and fractional time compare against an offset literal and a fractional-second literal, so a row carries each.", + "created either side of 2024-01-01": "recent keys off that date, and selected dates off 2020-01-01.", + "created null": "nullable date shift adds NULL to a date, and mixed temporal case falls through to observed.", + "q1 and q2 nulls": "complete asks whether both are non-null, case without else returns null for a null condition, and null logic folds one.", + "amount 0.1234567890123456": "precise threshold compares against 0.12345678901234567, so a value just below it is the only one that tests the comparison.", + "amount -7.5 and 0.0 and 1.0": "range predicate is BETWEEN -10 AND 10 with NOT IN (0, 1), and amount band is relative to MIN(amount).", + "ratio -1.0": "negative quotient divides by ratio + 1.0, so this row divides by zero. DuckDB and Databricks are expected to disagree here, and that disagreement is a finding to record rather than a value to pick around.", + "profile zip 9%": "dynamic match is `pattern LIKE profile.zip`, so the pattern comes from data and one row's zip has to contain a wildcard.", + "profile null and latitude null": "deep score reaches through two levels of struct, and postal length takes the length of a field that may be absent.", + "profile nick names with a space": "The field name carries a space, which is what makes quoting in the emitted SQL observable.", + "tags empty and null": "missing tags asks only about null, so an empty array must not answer the same way." + } + } + }, + "unrunnable": { + "note": "Definitions the corpus declares that cannot be executed against these rows on any engine, so neither the live comparison nor the DuckDB run attempts them. A key here is a finding, not a convenience: it records a definition whose emitted SQL does not run over a plausible column type.", + "survey::computed interval": "data-dict lowers `interval(amount + 1, days)` to `observed + (amount + 1 * INTERVAL '1 days')`, which DuckDB refuses to bind because amount is a double and there is no double-times-interval operator. amount has to be fractional for `precise threshold` to compare anything, so no single column type satisfies both, and this is the one that gives way. Upstream's own translation is what fails here, which makes it a data-dict question rather than a commons emitter one; raise it with commons kata g0ax." + } +} From 85067772c00a9870562297d92ba7362c1ebf4735 Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Wed, 9 Sep 2026 13:40:32 -0600 Subject: [PATCH 2/7] test(py): quote fixture identifiers, and describe through the merge Four review findings, all real. Relation references were built by joining the identifier parts raw, which breaks on both backends: `values` is a reserved word on Databricks, and Snowflake folds an unquoted name to upper case while the fixture tables are created under the authored spelling that the emitted definition SQL refers to its columns by. SQLAlchemy's identifier preparer does not fix it, because it quotes only where it judges quoting necessary and it judges both of those safe. The backends' own readers quote unconditionally, so the harness now uses them. The description test built a source with no dictionary, and a warehouse relation is only described when an authored table matched it, so every relation's columns were empty and the assertion could never have passed. It now passes the corpus dictionary the table was provisioned from, which also puts the authored-to-warehouse column binding under test, and reads the normalized `column` key rather than the raw reply's. The denied-table probe built its TableId positionally. TableId takes the table first and the catalog last, so a `catalog.schema.table` value arrived reversed and the probe tested a relation nobody configured. The driver dependencies are a uv group, not an extra, so the documented `uv sync --extra` would not have installed them. Refs kata whhh. --- pkg-py/tests/README.md | 2 +- pkg-py/tests/_live_warehouse.py | 52 +++++++++++++++++++++--- pkg-py/tests/test_live_warehouses.py | 60 ++++++++++++++++++---------- 3 files changed, 87 insertions(+), 27 deletions(-) diff --git a/pkg-py/tests/README.md b/pkg-py/tests/README.md index ba79ef647..7e09fecd0 100644 --- a/pkg-py/tests/README.md +++ b/pkg-py/tests/README.md @@ -13,7 +13,7 @@ Install the drivers, which are deliberately outside the `dev` group so an ordinary contributor does not install a warehouse client: ``` -uv sync --extra live-warehouse +uv sync --group live-warehouse ``` Point at a warehouse. The connection variables are the vendor's own, so one diff --git a/pkg-py/tests/_live_warehouse.py b/pkg-py/tests/_live_warehouse.py index 5fce89b42..e9bd2958a 100644 --- a/pkg-py/tests/_live_warehouse.py +++ b/pkg-py/tests/_live_warehouse.py @@ -27,6 +27,9 @@ import pytest import sqlalchemy +from commons._backends import EngineBackend +from commons._catalog import _databricks, _snowflake +from commons._data_source import TableId from tests._shared import load_shared_fixture __all__ = [ @@ -40,6 +43,11 @@ _ENV = load_shared_fixture("live-warehouse-env") _ROWS = load_shared_fixture("live-warehouse-rows") +_READER_QUOTING = { + "databricks": _databricks._quote_path, + "snowflake": _snowflake._quote_path, +} + # Doubles JSON cannot spell. The fixture carries them as strings. _SPECIAL_DOUBLES = {"NaN": math.nan, "Infinity": math.inf, "-Infinity": -math.inf} @@ -53,11 +61,45 @@ class LiveWarehouse: namespace: tuple[str, ...] fixtures: dict[str, str] - def table_id(self, table: str) -> tuple[str, ...]: - return (*self.namespace, table) - - def dotted(self, table: str) -> str: - return ".".join(self.table_id(table)) + @property + def engine_backend(self) -> EngineBackend: + return EngineBackend(self.engine) + + def table_id(self, table: str) -> TableId: + catalog, schema = self.namespace + return TableId(table=table, schema=schema, catalog=catalog) + + def quoted(self, table: str) -> str: + """A fixture table, quoted the way the backend's own reader quotes it. + + Joining the parts raw is wrong on both backends for different reasons: + `values` is a reserved word on Databricks, and Snowflake folds an + unquoted identifier to upper case while the fixture tables are created + under the authored spelling. SQLAlchemy's preparer does not solve + either, because it quotes only when it judges quoting necessary, and + it judged `main.commons_live.values` safe on Databricks and lowercase + Snowflake names safe as well. The readers quote unconditionally, and + they are what production uses. + """ + return _READER_QUOTING[self.backend](self.table_id(table).parts) + + def quoted_namespace(self) -> str: + return _READER_QUOTING[self.backend](list(self.namespace)) + + def fixture_table_id(self, name: str) -> TableId: + """A fixture variable's dotted value as a TableId. + + Spelled out rather than passed positionally: TableId takes the table + first and the catalog last, so `TableId(*value.split("."))` reverses a + `catalog.schema.table` value and probes something else entirely. + """ + parts = self.fixtures[name].split(".") + if len(parts) != 3: + raise ValueError( + f"{name} must name catalog.schema.table, got {self.fixtures[name]!r}." + ) + catalog, schema, table = parts + return TableId(table=table, schema=schema, catalog=catalog) def _missing(names: list[str]) -> list[str]: diff --git a/pkg-py/tests/test_live_warehouses.py b/pkg-py/tests/test_live_warehouses.py index baa8a5f9c..0ce1fd405 100644 --- a/pkg-py/tests/test_live_warehouses.py +++ b/pkg-py/tests/test_live_warehouses.py @@ -16,8 +16,8 @@ from __future__ import annotations import pytest +import yaml -from commons._backends import EngineBackend from commons._catalog import ( CatalogAuthorizationError, Selector, @@ -28,13 +28,14 @@ session_snapshot, ) from commons._catalog._security import probe_relation -from commons._data_source import TableId, data_source, list_tables +from commons._data_dictionary import DataDictionary +from commons._data_source import data_source, list_tables from tests._live_warehouse import ( duckdb_fixture_connection, matches, require_live_warehouse, ) -from tests._shared import load_shared_fixture +from tests._shared import SHARED_DIR, load_shared_fixture from tests.test_definition_emit_sql import compile_corpus ROWS_UNRUNNABLE = { @@ -50,10 +51,6 @@ _CASE = {"databricks": "lower", "snowflake": "upper"} -def _backend(live) -> EngineBackend: - return EngineBackend(live.engine) - - def _selector(live) -> Selector: return Selector(catalog=live.namespace[0], schema=live.namespace[1]) @@ -62,12 +59,20 @@ def _folded(live, name: str) -> str: return normalize_identifier(name, _CASE[live.backend]) +def _corpus_dictionary(filename: str) -> DataDictionary: + """The authored dictionary a fixture table was provisioned from.""" + path = SHARED_DIR / "definition-export" / "valid" / filename + return DataDictionary.model_validate( + yaml.safe_load(path.read_text(encoding="utf-8")) + ) + + @pytest.mark.parametrize("backend", BACKENDS) def test_the_listing_reports_the_provisioned_relations(backend): live = require_live_warehouse(backend) reader = _READERS[backend] - relations = reader.list_relations(_backend(live), _selector(live)) + relations = reader.list_relations(live.engine_backend, _selector(live)) listed = {_folded(live, relation.id.table) for relation in relations} assert {_folded(live, table) for table in FIXTURE_TABLES} <= listed @@ -87,19 +92,19 @@ def test_the_listing_rows_carry_the_keys_the_shared_fixture_pins(backend): live = require_live_warehouse(backend) pinned = load_shared_fixture("catalog-rows")[backend] catalog, schema = live.namespace - backend_obj = _backend(live) + backend_obj = live.engine_backend if backend == "databricks": relation_rows = backend_obj.query( "SELECT * FROM system.information_schema.tables " f"WHERE table_catalog = '{catalog}' AND table_schema = '{schema}'" ) - column_rows = backend_obj.query(f"DESCRIBE TABLE {live.dotted('survey')}") + column_rows = backend_obj.query(f"DESCRIBE TABLE {live.quoted('survey')}") else: relation_rows = backend_obj.query( - f'SHOW OBJECTS IN SCHEMA "{catalog}"."{schema}"' + f"SHOW OBJECTS IN SCHEMA {live.quoted_namespace()}" ) - column_rows = backend_obj.query(f"DESC TABLE {live.dotted('survey')}") + column_rows = backend_obj.query(f"DESC TABLE {live.quoted('survey')}") for group, rows in (("relations", relation_rows), ("columns", column_rows)): assert rows, f"{backend} returned no {group}" @@ -114,9 +119,18 @@ def test_the_listing_rows_carry_the_keys_the_shared_fixture_pins(backend): @pytest.mark.parametrize("backend", BACKENDS) def test_a_source_over_the_namespace_describes_every_fixture_table(backend): + """The listing, and what describing the relations against it reports. + + The dictionary is the corpus one the survey table was provisioned from, + which is what makes this the description path rather than the listing + path: a warehouse relation is only described when an authored table + matched it, so a source built without a dictionary leaves every relation's + columns empty and there would be nothing here to assert. + """ live = require_live_warehouse(backend) + dictionary = _corpus_dictionary("language.yaml") - source = data_source(live.engine, tables=_selector(live)) + source = data_source(live.engine, tables=_selector(live), dictionary=dictionary) labels = {_folded(live, label) for label in list_tables(source)} assert {_folded(live, table) for table in FIXTURE_TABLES} <= labels @@ -127,9 +141,8 @@ def test_a_source_over_the_namespace_describes_every_fixture_table(backend): ) assert source.relations is not None columns = source.relations[survey].columns - described = { - str(column.get("col_name") or column.get("name")).lower() for column in columns - } + assert columns, "the relation was never described" + described = {str(column["column"]).lower() for column in columns} authored = { column["name"] for column in load_shared_fixture("live-warehouse-rows")["tables"]["survey"][ @@ -137,13 +150,18 @@ def test_a_source_over_the_namespace_describes_every_fixture_table(backend): ] } assert authored <= described + # Every authored column bound to a real one, which is what the merge + # promises and what a case-folding mistake would break. + assert source.dictionary is not None + merged = source.dictionary.tables[survey].columns + assert authored <= {name.lower() for name in merged} @pytest.mark.parametrize("backend", BACKENDS) def test_the_session_snapshot_reads_the_live_identity(backend): live = require_live_warehouse(backend) - snapshot = session_snapshot(_backend(live)) + snapshot = session_snapshot(live.engine_backend) assert snapshot is not None assert snapshot.principal @@ -164,8 +182,8 @@ def test_a_refusal_is_classified_as_authorization(backend): classify is a defect in that file, not in the warehouse. """ live = require_live_warehouse(backend, "denied_table") - denied = TableId(*live.fixtures["denied_table"].split(".")) - backend_obj = _backend(live) + denied = live.fixture_table_id("denied_table") + backend_obj = live.engine_backend probe = probe_relation(backend_obj, denied) @@ -195,7 +213,7 @@ def test_the_emitted_definition_sql_agrees_with_duckdb(backend): live = require_live_warehouse(backend) duckdb_con = duckdb_fixture_connection() rows_fixture = load_shared_fixture("live-warehouse-rows") - backend_obj = _backend(live) + backend_obj = live.engine_backend mixed_grain = load_shared_fixture("definitions")["mixed_grain"] unrunnable = set(ROWS_UNRUNNABLE) @@ -217,7 +235,7 @@ def test_the_emitted_definition_sql_agrees_with_duckdb(backend): warehouse_values = [ next(iter(row.values())) for row in backend_obj.query( - f"SELECT {record.sql} AS result FROM {live.dotted(table)}" + f"SELECT {record.sql} AS result FROM {live.quoted(table)}" ) ] executed += 1 From 239b886c22091fe144e463b11b2649a7336cb471 Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Tue, 20 Oct 2026 13:22:44 -0600 Subject: [PATCH 3/7] test(py): sign in to live warehouses non-interactively, and run them in CI Snowflake signs in with a key pair from SNOWFLAKE_PRIVATE_KEY_FILE, the variable the Snowflake CLI and snowflakeauth read. Service users cannot use passwords, so SNOWFLAKE_PASSWORD leaves the contract. Databricks credentials now come from the SDK's unified authentication, which resolves a token, a service principal's client secret, GitHub Actions OIDC, or a CLI login from the environment. That replaces the harness's own choice between a token and the browser flow, and it covers whichever credential the CI service principal ends up with. GitHub OIDC needs databricks-sdk 0.55. The new workflow runs each backend from its own GitHub environment on pushes to main and on manual dispatch. It checks the contract's required variables first: none set skips the job, and some set fails it, so a misconfigured backend cannot pass by skipping every test. --- .github/workflows/py-live-warehouse.yaml | 96 +++++++++++++++++++ pkg-py/pyproject.toml | 6 +- pkg-py/tests/README.md | 25 +++-- pkg-py/tests/_live_warehouse.py | 36 +++---- pkg-py/uv.lock | 51 ++++++++++ .../fixtures/shared/live-warehouse-env.json | 13 ++- tests/shared/live-warehouse-env.json | 13 ++- 7 files changed, 206 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/py-live-warehouse.yaml diff --git a/.github/workflows/py-live-warehouse.yaml b/.github/workflows/py-live-warehouse.yaml new file mode 100644 index 000000000..d86e6ee17 --- /dev/null +++ b/.github/workflows/py-live-warehouse.yaml @@ -0,0 +1,96 @@ +name: "Py: Live warehouse tests" + +on: + push: + branches: [main] + paths: [pkg-py/**, tests/shared/**, .github/workflows/py-live-warehouse.yaml] + workflow_dispatch: + +permissions: + contents: read + +jobs: + live: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + backend: [databricks, snowflake] + # Each backend's credentials live in its own environment. The Databricks + # federation policy trusts this repository's `databricks-ci` environment + # by name. + environment: ${{ matrix.backend }}-ci + permissions: + contents: read + # GitHub OIDC for the Databricks service principal. + id-token: write + defaults: + run: + working-directory: pkg-py + env: + UV_PYTHON: "3.13" + DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} + DATABRICKS_HTTP_PATH: ${{ vars.DATABRICKS_HTTP_PATH }} + DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} + # Unset unless the federation policy accepts an audience other than the + # workspace's token endpoint, which is what the SDK asks GitHub for. + DATABRICKS_TOKEN_AUDIENCE: ${{ vars.DATABRICKS_TOKEN_AUDIENCE }} + # Unset when the service principal signs in with GitHub OIDC instead. + DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }} + COMMONS_TEST_DATABRICKS_SCHEMA: ${{ vars.COMMONS_TEST_DATABRICKS_SCHEMA }} + COMMONS_TEST_DATABRICKS_DENIED_TABLE: ${{ vars.COMMONS_TEST_DATABRICKS_DENIED_TABLE }} + SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} + SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} + COMMONS_TEST_SNOWFLAKE_SCHEMA: ${{ vars.COMMONS_TEST_SNOWFLAKE_SCHEMA }} + COMMONS_TEST_SNOWFLAKE_DENIED_TABLE: ${{ vars.COMMONS_TEST_SNOWFLAKE_DENIED_TABLE }} + COMMONS_TEST_SNOWFLAKE_ALTERNATE_ROLE: ${{ vars.COMMONS_TEST_SNOWFLAKE_ALTERNATE_ROLE }} + steps: + - uses: actions/checkout@v6 + - name: Write the Snowflake private key + if: matrix.backend == 'snowflake' + env: + SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} + run: | + if [ -n "$SNOWFLAKE_PRIVATE_KEY" ]; then + key="$RUNNER_TEMP/snowflake_key.p8" + (umask 077 && printf '%s\n' "$SNOWFLAKE_PRIVATE_KEY" > "$key") + echo "SNOWFLAKE_PRIVATE_KEY_FILE=$key" >> "$GITHUB_ENV" + fi + - name: Check the configuration + # The tests skip when a required variable is unset, so a job with a + # missing variable would pass having tested nothing. Stop here instead: + # skip the job when nothing is configured, and fail it when only some + # of it is. + id: configured + run: | + backend='${{ matrix.backend }}' + names=$(jq -r --arg b "$backend" \ + '.backends[$b] | .connection.required[], .fixtures.schema.variable' \ + ../tests/shared/live-warehouse-env.json) + missing=() + for name in $names; do + [ -n "${!name:-}" ] || missing+=("$name") + done + total=$(echo "$names" | wc -w) + if [ "${#missing[@]}" -eq "$total" ]; then + echo "::notice::$backend is not configured for this repository; skipping." + echo "run=false" >> "$GITHUB_OUTPUT" + elif [ "${#missing[@]}" -gt 0 ]; then + echo "::error::$backend is partly configured. Missing: ${missing[*]}" + exit 1 + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + - uses: astral-sh/setup-uv@v6 + if: steps.configured.outputs.run == 'true' + with: + enable-cache: true + cache-dependency-glob: pkg-py/uv.lock + - name: Install dependencies + if: steps.configured.outputs.run == 'true' + run: uv sync --locked --all-extras --group live-warehouse + - name: Test + if: steps.configured.outputs.run == 'true' + run: uv run pytest tests/test_live_warehouses.py -k '${{ matrix.backend }}' -rs diff --git a/pkg-py/pyproject.toml b/pkg-py/pyproject.toml index bbd6047ba..128b32314 100644 --- a/pkg-py/pyproject.toml +++ b/pkg-py/pyproject.toml @@ -52,7 +52,11 @@ docs = [ # Drivers for the live warehouse tests, which skip without a configured # warehouse. Kept out of `dev` so an ordinary contributor does not install a # warehouse client to run the suite. -live-warehouse = ["databricks-sqlalchemy>=2", "snowflake-sqlalchemy>=1.6"] +live-warehouse = [ + "databricks-sdk>=0.55", + "databricks-sqlalchemy>=2", + "snowflake-sqlalchemy>=1.6", +] [build-system] requires = ["hatchling"] diff --git a/pkg-py/tests/README.md b/pkg-py/tests/README.md index 7e09fecd0..9cc41a233 100644 --- a/pkg-py/tests/README.md +++ b/pkg-py/tests/README.md @@ -17,19 +17,27 @@ uv sync --group live-warehouse ``` Point at a warehouse. The connection variables are the vendor's own, so one -export block also serves the Databricks CLI and the R suite: +export block also serves the vendor CLIs and the R suite: ``` export DATABRICKS_HOST=https://example.cloud.databricks.com export DATABRICKS_HTTP_PATH=/sql/1.0/warehouses/abc123 -export DATABRICKS_TOKEN=dapi... # omit for the OAuth browser flow export COMMONS_TEST_DATABRICKS_SCHEMA=main.commons_live + +export SNOWFLAKE_ACCOUNT=myorg-myaccount +export SNOWFLAKE_USER=MY_USER +export SNOWFLAKE_WAREHOUSE=MY_WH +export SNOWFLAKE_PRIVATE_KEY_FILE=~/.snowflake/keys/rsa_key.p8 +export COMMONS_TEST_SNOWFLAKE_SCHEMA=COMMONS_CI.PUBLIC ``` -With `DATABRICKS_TOKEN` unset the connector runs the OAuth user-to-machine -flow, which opens a browser and reuses an SSO session, so an SSO-only account -needs no long-lived secret. `tests/shared/live-warehouse-env.json` is the full -list of variables, including the optional ones that gate individual tests. +Databricks credentials come from the SDK's +[unified authentication](https://docs.databricks.com/aws/en/dev-tools/auth/unified-auth): +`DATABRICKS_TOKEN`, a service principal's `DATABRICKS_CLIENT_ID` and +`DATABRICKS_CLIENT_SECRET`, or a `databricks auth login` session. Snowflake +signs in with a key pair registered to `SNOWFLAKE_USER`. +`tests/shared/live-warehouse-env.json` is the full list of variables, +including the optional ones that gate individual tests. Create the tables the tests expect: @@ -48,4 +56,7 @@ access-refusal test skips. `test_live_warehouse_fixture.py` covers the half of this that needs no warehouse: the rows load, every corpus definition runs against them, and the -provisioning SQL renders. It runs in CI. +provisioning SQL renders. It runs in CI with the rest of the suite. +`.github/workflows/py-live-warehouse.yaml` runs the live tests themselves, +with credentials from the repository's `databricks-ci` and `snowflake-ci` +environments, and skips a backend whose environment is not configured. diff --git a/pkg-py/tests/_live_warehouse.py b/pkg-py/tests/_live_warehouse.py index e9bd2958a..f7a8e857d 100644 --- a/pkg-py/tests/_live_warehouse.py +++ b/pkg-py/tests/_live_warehouse.py @@ -134,36 +134,35 @@ def require_live_warehouse(backend: str, *needs: str) -> LiveWarehouse: f"{expected} identifier parts, got {fixtures['schema']!r}." ) - engine = _engine(backend, connection, namespace) + engine = _engine(backend, namespace) return LiveWarehouse( backend=backend, engine=engine, namespace=namespace, fixtures=fixtures ) -def _engine( - backend: str, connection: dict[str, Any], namespace: tuple[str, ...] -) -> sqlalchemy.Engine: +def _engine(backend: str, namespace: tuple[str, ...]) -> sqlalchemy.Engine: if backend == "databricks": pytest.importorskip( "databricks.sqlalchemy", reason="pip install databricks-sqlalchemy" ) + from databricks.sdk.core import Config + host = os.environ["DATABRICKS_HOST"].removeprefix("https://").rstrip("/") - token = os.environ.get(connection["token"], "") catalog, schema = namespace - query = {"http_path": os.environ["DATABRICKS_HTTP_PATH"]} url = sqlalchemy.engine.URL.create( "databricks", - username="token" if token else None, - password=token or None, host=host, database=schema, - query={**query, "catalog": catalog}, + query={"http_path": os.environ["DATABRICKS_HTTP_PATH"], "catalog": catalog}, + ) + # The SDK's unified authentication picks the credential from the + # environment: a token, a service principal's secret or GitHub OIDC + # token, or a CLI login. The connector asks for headers per request, + # and the SDK refreshes short-lived tokens behind them. + config = Config(host=f"https://{host}") + return sqlalchemy.create_engine( + url, connect_args={"credentials_provider": lambda: config.authenticate} ) - # Without a token the connector runs the OAuth user-to-machine flow, - # which opens a browser and reuses the SSO session. That is what makes - # an SSO-only account usable without minting a long-lived secret. - connect_args = {} if token else {"auth_type": "databricks-oauth"} - return sqlalchemy.create_engine(url, connect_args=connect_args) pytest.importorskip( "snowflake.sqlalchemy", reason="pip install snowflake-sqlalchemy" @@ -172,12 +171,17 @@ def _engine( url = sqlalchemy.engine.URL.create( "snowflake", username=os.environ["SNOWFLAKE_USER"], - password=os.environ.get(connection["token"]) or None, host=os.environ["SNOWFLAKE_ACCOUNT"], database=f"{database}/{schema}", query={"warehouse": os.environ["SNOWFLAKE_WAREHOUSE"]}, ) - return sqlalchemy.create_engine(url) + return sqlalchemy.create_engine( + url, + connect_args={ + "authenticator": "SNOWFLAKE_JWT", + "private_key_file": os.environ["SNOWFLAKE_PRIVATE_KEY_FILE"], + }, + ) def _duckdb_value(value: Any, type_name: str) -> Any: diff --git a/pkg-py/uv.lock b/pkg-py/uv.lock index aabf96dae..a8da40a72 100644 --- a/pkg-py/uv.lock +++ b/pkg-py/uv.lock @@ -592,6 +592,7 @@ docs = [ { name = "great-docs" }, ] live-warehouse = [ + { name = "databricks-sdk" }, { name = "databricks-sqlalchemy" }, { name = "snowflake-sqlalchemy" }, ] @@ -636,6 +637,7 @@ dev = [ ] docs = [{ name = "great-docs", specifier = ">=0.17" }] live-warehouse = [ + { name = "databricks-sdk", specifier = ">=0.55" }, { name = "databricks-sqlalchemy", specifier = ">=2" }, { name = "snowflake-sqlalchemy", specifier = ">=1.6" }, ] @@ -908,6 +910,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/84/582e1daf3f2eb296573c11b774abf10223e9858d4c5c5c936180b643a720/databackend-0.0.3-py3-none-any.whl", hash = "sha256:2ded97ff85a0ca272d752fbc0513f70512d2c23325e3c72de15613df7e959ef5", size = 7001, upload-time = "2024-08-21T21:09:41.137Z" }, ] +[[package]] +name = "databricks-sdk" +version = "0.154.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "protobuf" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/37/40bc3745edff2e3ab66cb3adc392bf38cde987f590f2c38b849cb1f0734a/databricks_sdk-0.154.0.tar.gz", hash = "sha256:70af90c889e76ecc0ab62d5df5b2d7c9dca4f4e7bc2caddec3701c47c1719d79", size = 1162776, upload-time = "2026-10-19T04:36:46.235Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/31/5d6f93251dc6c9a2155318182a85e6884f3aeb37c3ea3109d10f77f84f3d/databricks_sdk-0.154.0-py3-none-any.whl", hash = "sha256:a87413ec9469c278104999b1a9f644d52aa97f2307be8f70252eaae3e6e74931", size = 1105420, upload-time = "2026-10-19T04:36:43.767Z" }, +] + [[package]] name = "databricks-sql-connector" version = "4.6.0" @@ -1163,6 +1180,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/e0/014d5d9d7a4564cf1c40b5039bc882db69fd881111e03ab3657ac0b218e2/fsspec-2025.7.0-py3-none-any.whl", hash = "sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21", size = 199597, upload-time = "2025-07-15T16:05:19.529Z" }, ] +[[package]] +name = "google-auth" +version = "2.62.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/b9/4b2528f30114b106e3c3a7298ba38d85e682fd1f83fe7c4a5b77c7677082/google_auth-2.62.0.tar.gz", hash = "sha256:0bef0ce54bdf9ce226c5d66e4264413bd918141c31bbe49fb52eac882f513d69", size = 398180, upload-time = "2026-10-12T19:20:48.328Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/a3/87fe9b7203ec2e56ea6a57c75e6d632eaf9487c7f7978cdad1d801bee0af/google_auth-2.62.0-py3-none-any.whl", hash = "sha256:4ff4319aeb4ad128409759d397a9fcafad126d0031d241cc0dd6b9a00b43e3f3", size = 267911, upload-time = "2026-10-12T19:20:46.355Z" }, +] + [[package]] name = "google-re2" version = "1.1.20251105" @@ -3533,6 +3563,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/c0/37d4a7e8e2f7a6076283673d5298018ca26478b934c6ee369e10505ab32c/pyarrow-25.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4288f27577352d608ca08553b0865e4a9b3aa14820c5d95b53337218d609835b", size = 28753071, upload-time = "2026-08-10T12:40:46.623Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pybreaker" version = "1.4.1" diff --git a/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json b/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json index 8540b5396..771bc3398 100644 --- a/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json +++ b/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json @@ -1,10 +1,9 @@ { - "note": "The environment both suites read to reach a real warehouse. Held here so the two harnesses cannot drift on a variable name. Connection variables are the vendor's own, which is what the Databricks and Snowflake client libraries read in Python and what odbc::databricks() and odbc::snowflake() read in R, so one export block serves both languages and the vendor CLIs. Fixture variables carry one dotted identifier each. A backend whose required variables are unset skips; an optional fixture skips only the tests that need it.", + "note": "The environment both suites read to reach a real warehouse. Held here so the two harnesses cannot drift on a variable name. Connection variables are the vendor's own, so one export block serves both languages and the vendor CLIs. Databricks credentials are left to the vendor's unified authentication, which resolves DATABRICKS_TOKEN, a DATABRICKS_CLIENT_ID with a secret or GitHub Actions OIDC, or a Databricks CLI login, so none is named here. Snowflake signs in with a key pair, the one method a service user and a person share. Fixture variables carry one dotted identifier each. A backend whose required variables are unset skips; an optional fixture skips only the tests that need it.", "backends": { "databricks": { "connection": { - "required": ["DATABRICKS_HOST", "DATABRICKS_HTTP_PATH"], - "token": "DATABRICKS_TOKEN" + "required": ["DATABRICKS_HOST", "DATABRICKS_HTTP_PATH"] }, "fixtures": { "schema": { @@ -29,8 +28,12 @@ }, "snowflake": { "connection": { - "required": ["SNOWFLAKE_ACCOUNT", "SNOWFLAKE_USER", "SNOWFLAKE_WAREHOUSE"], - "token": "SNOWFLAKE_PASSWORD" + "required": [ + "SNOWFLAKE_ACCOUNT", + "SNOWFLAKE_USER", + "SNOWFLAKE_WAREHOUSE", + "SNOWFLAKE_PRIVATE_KEY_FILE" + ] }, "fixtures": { "schema": { diff --git a/tests/shared/live-warehouse-env.json b/tests/shared/live-warehouse-env.json index 8540b5396..771bc3398 100644 --- a/tests/shared/live-warehouse-env.json +++ b/tests/shared/live-warehouse-env.json @@ -1,10 +1,9 @@ { - "note": "The environment both suites read to reach a real warehouse. Held here so the two harnesses cannot drift on a variable name. Connection variables are the vendor's own, which is what the Databricks and Snowflake client libraries read in Python and what odbc::databricks() and odbc::snowflake() read in R, so one export block serves both languages and the vendor CLIs. Fixture variables carry one dotted identifier each. A backend whose required variables are unset skips; an optional fixture skips only the tests that need it.", + "note": "The environment both suites read to reach a real warehouse. Held here so the two harnesses cannot drift on a variable name. Connection variables are the vendor's own, so one export block serves both languages and the vendor CLIs. Databricks credentials are left to the vendor's unified authentication, which resolves DATABRICKS_TOKEN, a DATABRICKS_CLIENT_ID with a secret or GitHub Actions OIDC, or a Databricks CLI login, so none is named here. Snowflake signs in with a key pair, the one method a service user and a person share. Fixture variables carry one dotted identifier each. A backend whose required variables are unset skips; an optional fixture skips only the tests that need it.", "backends": { "databricks": { "connection": { - "required": ["DATABRICKS_HOST", "DATABRICKS_HTTP_PATH"], - "token": "DATABRICKS_TOKEN" + "required": ["DATABRICKS_HOST", "DATABRICKS_HTTP_PATH"] }, "fixtures": { "schema": { @@ -29,8 +28,12 @@ }, "snowflake": { "connection": { - "required": ["SNOWFLAKE_ACCOUNT", "SNOWFLAKE_USER", "SNOWFLAKE_WAREHOUSE"], - "token": "SNOWFLAKE_PASSWORD" + "required": [ + "SNOWFLAKE_ACCOUNT", + "SNOWFLAKE_USER", + "SNOWFLAKE_WAREHOUSE", + "SNOWFLAKE_PRIVATE_KEY_FILE" + ] }, "fixtures": { "schema": { From c7c601c8249c5a75b645a37be298d306cca45f6c Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Tue, 20 Oct 2026 13:24:22 -0600 Subject: [PATCH 4/7] test: grant the live-test principal its tables, and create the one it is refused provision-live-warehouse.py takes --grant-to, naming the Snowflake role or Databricks service principal the tests connect as. The SQL then grants it usage on the namespace and SELECT on each fixture table, and creates denied_orders, a copy of orders with no grant. Whoever runs the SQL owns the copy and the test principal does not, which is the second principal the denied-table fixture always needed. Without --grant-to the output is unchanged and the denied table is still left to do by hand. Databricks USE CATALOG stays with the catalog's owner, so the SQL says so rather than granting it. --- pkg-py/tests/README.md | 9 +-- pkg-py/tests/test_live_warehouse_fixture.py | 18 +++++ scripts/provision-live-warehouse.py | 79 ++++++++++++++++----- 3 files changed, 86 insertions(+), 20 deletions(-) diff --git a/pkg-py/tests/README.md b/pkg-py/tests/README.md index 9cc41a233..5c05bab11 100644 --- a/pkg-py/tests/README.md +++ b/pkg-py/tests/README.md @@ -49,10 +49,11 @@ That prints SQL to run; it does not connect. The rows come from `tests/shared/live-warehouse-rows.json`, and the same file is loaded into DuckDB so the two engines' answers can be compared. -One fixture it cannot create is the denied table, since an owner can always -read their own relation. Making one needs a second principal, and the -generated SQL says what is left to do. Until its variable is set, the -access-refusal test skips. +The access-refusal test needs a table the test principal cannot read. Pass +`--grant-to` with the principal the tests connect as (a Snowflake role, or a +Databricks service principal's application ID), and the SQL grants it the +fixture tables and creates a copy it is refused. Without `--grant-to` there is +no such table, so that test skips. `test_live_warehouse_fixture.py` covers the half of this that needs no warehouse: the rows load, every corpus definition runs against them, and the diff --git a/pkg-py/tests/test_live_warehouse_fixture.py b/pkg-py/tests/test_live_warehouse_fixture.py index fead2da87..e7c4c5867 100644 --- a/pkg-py/tests/test_live_warehouse_fixture.py +++ b/pkg-py/tests/test_live_warehouse_fixture.py @@ -128,6 +128,24 @@ def test_the_provisioning_script_renders_every_fixture_table(dialect): assert literal in sql, literal +@pytest.mark.parametrize("dialect", ["databricks", "snowflake"]) +def test_the_grants_cover_every_fixture_table_but_the_denied_one(dialect): + generator = _generator() + sql = generator.render(dialect, "catalog.schema", grantee="tester") + granted = [line for line in sql.splitlines() if line.startswith("GRANT SELECT")] + + assert len(granted) == len(ROWS["tables"]) + for table in ROWS["tables"]: + assert any( + generator.quote_identifier(table, dialect) in line for line in granted + ) + assert not any(generator.DENIED_TABLE in line for line in granted) + assert ( + f".{generator.quote_identifier(generator.DENIED_TABLE, dialect)} AS SELECT" + in sql + ) + + def test_a_mismatch_is_reported_as_one(): """The comparator, since every live assertion rests on it.""" assert matches([1.0, None], [None, 1.0]) diff --git a/scripts/provision-live-warehouse.py b/scripts/provision-live-warehouse.py index 47e29b404..5bfb6e5e5 100755 --- a/scripts/provision-live-warehouse.py +++ b/scripts/provision-live-warehouse.py @@ -7,10 +7,14 @@ to run, which leaves no committed artifact to go stale. python3 scripts/provision-live-warehouse.py databricks main.commons_live - -The denied-table fixture is not created here. A relation its owner cannot read -does not exist, so making one needs a second principal; the script prints what -remains to be done by hand. + python3 scripts/provision-live-warehouse.py snowflake COMMONS_CI.PUBLIC --grant-to COMMONS_CI_ROLE + +`--grant-to` names the principal the tests connect as: a Snowflake role, or a +Databricks service principal's application ID. Its grants cover the fixture +tables and leave out a copy of one, which becomes the denied-table fixture. +That works because whoever runs this SQL owns the copy and the test principal +does not. Without `--grant-to` the tests run as the owner, who can read +everything, so the denied table is left to do by hand. """ from __future__ import annotations @@ -20,7 +24,10 @@ from pathlib import Path from typing import Any -FIXTURE = Path(__file__).resolve().parent.parent / "tests" / "shared" / "live-warehouse-rows.json" +SHARED = Path(__file__).resolve().parent.parent / "tests" / "shared" +FIXTURE = SHARED / "live-warehouse-rows.json" +ENV = SHARED / "live-warehouse-env.json" +DENIED_TABLE = "denied_orders" SPECIAL_DOUBLES = { "databricks": {"NaN": "double('NaN')", "Infinity": "double('Infinity')", "-Infinity": "double('-Infinity')"}, @@ -102,7 +109,7 @@ def quote_identifier(name: str, dialect: str) -> str: return '"' + name.replace('"', '""') + '"' -def render(dialect: str, namespace: str) -> str: +def render(dialect: str, namespace: str, grantee: str | None = None) -> str: fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) types = fixture["types"] parts = namespace.split(".") @@ -143,22 +150,62 @@ def render(dialect: str, namespace: str) -> str: ) lines += [f"INSERT INTO {target} ({column_list}) VALUES", values + ";", ""] - lines += [ - "-- Still to do by hand, because an owner can always read their own relation:", - f"-- Create a copy of {qualified}.{quote_identifier('orders', dialect)} owned by another", - "-- principal, in a schema this one has no SELECT on, and export its identifier as the", - "-- denied-table variable named in tests/shared/live-warehouse-env.json. Until then the", - "-- access-refusal tests skip.", - ] + if grantee is None: + lines += [ + "-- Still to do by hand, because an owner can always read their own relation:", + f"-- Create a copy of {qualified}.{quote_identifier('orders', dialect)} owned by another", + "-- principal, in a schema this one has no SELECT on, and export its identifier as the", + "-- denied-table variable named in tests/shared/live-warehouse-env.json. Until then the", + "-- access-refusal tests skip.", + ] + else: + lines += grants(dialect, parts, qualified, list(fixture["tables"]), grantee) return "\n".join(lines) + "\n" +def grants(dialect: str, parts: list[str], qualified: str, tables: list[str], grantee: str) -> list[str]: + """Read access for the test principal, and a table it is refused.""" + principal = quote_identifier(grantee, dialect) + denied = f"{qualified}.{quote_identifier(DENIED_TABLE, dialect)}" + lines = [ + f"DROP TABLE IF EXISTS {denied};", + f"CREATE TABLE {denied} AS SELECT * FROM {qualified}.{quote_identifier('orders', dialect)};", + "", + ] + if dialect == "databricks": + lines += [ + f"-- {principal} also needs USE CATALOG on {quote_identifier(parts[0], dialect)}, from its owner.", + f"GRANT USE SCHEMA ON SCHEMA {qualified} TO {principal};", + ] + lines += [f"GRANT SELECT ON TABLE {qualified}.{quote_identifier(t, dialect)} TO {principal};" for t in tables] + else: + lines += [ + f"GRANT USAGE ON DATABASE {quote_identifier(parts[0], dialect)} TO ROLE {principal};", + f"GRANT USAGE ON SCHEMA {qualified} TO ROLE {principal};", + ] + lines += [f"GRANT SELECT ON TABLE {qualified}.{quote_identifier(t, dialect)} TO ROLE {principal};" for t in tables] + variable = json.loads(ENV.read_text(encoding="utf-8"))["backends"][dialect]["fixtures"]["denied_table"]["variable"] + return lines + ["", f"-- Then: export {variable}={'.'.join(parts)}.{DENIED_TABLE}"] + + def main(argv: list[str]) -> int: - if len(argv) != 3 or argv[1] not in ("databricks", "snowflake"): + args = argv[1:] + grantee = None + if "--grant-to" in args: + at = args.index("--grant-to") + if at + 1 >= len(args): + args = [] + else: + grantee = args[at + 1] + del args[at : at + 2] + if len(args) != 2 or args[0] not in ("databricks", "snowflake"): print(__doc__, file=sys.stderr) - print("Usage: provision-live-warehouse.py {databricks|snowflake} ", file=sys.stderr) + print( + "Usage: provision-live-warehouse.py {databricks|snowflake} [--grant-to ]", + file=sys.stderr, + ) return 2 - print(render(argv[1], argv[2]), end="") + print(render(args[0], args[1], grantee), end="") return 0 From a9a3bc145ab1c0cfab7038ffc124e006bdcfeb0c Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Tue, 20 Oct 2026 13:26:01 -0600 Subject: [PATCH 5/7] test: say in the generated SQL that the test principal must not run it A principal that runs the grants owns denied_orders and can read it, which turns the access-refusal test into a failure. --- scripts/provision-live-warehouse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/provision-live-warehouse.py b/scripts/provision-live-warehouse.py index 5bfb6e5e5..c2cd6e785 100755 --- a/scripts/provision-live-warehouse.py +++ b/scripts/provision-live-warehouse.py @@ -168,6 +168,8 @@ def grants(dialect: str, parts: list[str], qualified: str, tables: list[str], gr principal = quote_identifier(grantee, dialect) denied = f"{qualified}.{quote_identifier(DENIED_TABLE, dialect)}" lines = [ + f"-- Run this as a principal other than {principal}. An owner can read its own", + f"-- tables, so {principal} must not own {denied}.", f"DROP TABLE IF EXISTS {denied};", f"CREATE TABLE {denied} AS SELECT * FROM {qualified}.{quote_identifier('orders', dialect)};", "", From 8a3a21f3baf9bc91054ca6013c08f8f556f49532 Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Tue, 20 Oct 2026 13:47:24 -0600 Subject: [PATCH 6/7] test: say the provisioning script creates the denied table Its --grant-to option creates one owned by whoever runs the SQL, so the contract no longer needs to call the fixture something provisioning cannot finish. --- pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json | 2 +- tests/shared/live-warehouse-env.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json b/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json index 771bc3398..8caa76f97 100644 --- a/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json +++ b/pkg-r/tests/testthat/fixtures/shared/live-warehouse-env.json @@ -16,7 +16,7 @@ "variable": "COMMONS_TEST_DATABRICKS_DENIED_TABLE", "required": false, "arity": 3, - "description": "A relation the test principal genuinely cannot read. Needs an owner other than the test principal, so provisioning cannot finish it." + "description": "A relation the test principal genuinely cannot read. Provisioning with --grant-to creates one, owned by whoever runs the SQL." }, "metric_view": { "variable": "COMMONS_TEST_DATABRICKS_METRIC_VIEW", diff --git a/tests/shared/live-warehouse-env.json b/tests/shared/live-warehouse-env.json index 771bc3398..8caa76f97 100644 --- a/tests/shared/live-warehouse-env.json +++ b/tests/shared/live-warehouse-env.json @@ -16,7 +16,7 @@ "variable": "COMMONS_TEST_DATABRICKS_DENIED_TABLE", "required": false, "arity": 3, - "description": "A relation the test principal genuinely cannot read. Needs an owner other than the test principal, so provisioning cannot finish it." + "description": "A relation the test principal genuinely cannot read. Provisioning with --grant-to creates one, owned by whoever runs the SQL." }, "metric_view": { "variable": "COMMONS_TEST_DATABRICKS_METRIC_VIEW", From a1b0faf1d4e4afde6777a01f72e88947d73d32bb Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Tue, 20 Oct 2026 15:09:21 -0600 Subject: [PATCH 7/7] test(py): import the Databricks SDK through importorskip The check job installs every extra but not the live-warehouse group, so a static import of databricks.sdk.core failed pyrefly there, and would have errored rather than skipped where the SDK is missing. --- pkg-py/tests/_live_warehouse.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg-py/tests/_live_warehouse.py b/pkg-py/tests/_live_warehouse.py index f7a8e857d..b2fc94a6c 100644 --- a/pkg-py/tests/_live_warehouse.py +++ b/pkg-py/tests/_live_warehouse.py @@ -145,7 +145,9 @@ def _engine(backend: str, namespace: tuple[str, ...]) -> sqlalchemy.Engine: pytest.importorskip( "databricks.sqlalchemy", reason="pip install databricks-sqlalchemy" ) - from databricks.sdk.core import Config + Config = pytest.importorskip( + "databricks.sdk.core", reason="pip install databricks-sdk" + ).Config host = os.environ["DATABRICKS_HOST"].removeprefix("https://").rstrip("/") catalog, schema = namespace