From 6cd19681b67ff4727afef203e428b6becf058fae Mon Sep 17 00:00:00 2001 From: Zhao vistart Date: Mon, 9 Nov 2026 16:17:01 +0800 Subject: [PATCH 1/4] feat(snowflake): add routines, sequences and catalog-object introspection --- docs/introspection.md | 49 +- .../backend/impl/snowflake/cli/README.md | 32 +- .../backend/impl/snowflake/cli/introspect.py | 117 ++- .../snowflake/expression/introspection.py | 140 ++- .../impl/snowflake/introspection/__init__.py | 4 + .../snowflake/introspection/introspector.py | 820 ++++++++++++++++ .../impl/snowflake/introspection/types.py | 79 +- .../impl/snowflake/mixins/introspection.py | 469 ++++++++- .../test_introspect_catalog_families_cli.py | 440 +++++++++ .../test_routines_catalog_identity.py | 917 ++++++++++++++++++ 10 files changed, 3052 insertions(+), 15 deletions(-) create mode 100644 tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_catalog_families_cli.py create mode 100644 tests/rhosocial/activerecord_snowflake_test/feature/backend/introspection/test_routines_catalog_identity.py diff --git a/docs/introspection.md b/docs/introspection.md index 523fe97..d09e2ed 100644 --- a/docs/introspection.md +++ b/docs/introspection.md @@ -13,6 +13,9 @@ The introspection system is available via `backend.introspector` and answers: - **Index information**: primary-key and unique constraints, which are as close to an index as this engine has - **Foreign-key information**: referenced table, column mapping, and the update/delete rules the catalogue declares - **View information**: definition, check option, updatability +- **Routine information**: stored procedures and user-defined functions, with language, argument signature, return type, owner and comment +- **Sequence information**: standalone sequences, with start, increment, next value and ordered property +- **Catalog objects**: named stages, file formats, pipes, tasks and streams, each with its owning role and engine-specific facts - **Storage information**: the four billed storage tiers of one table, and the total that fills a table record's `size_bytes` - **Data answers**: per-column profiles, value listings, and two columns' domain overlap @@ -22,7 +25,7 @@ Snowflake has no trigger object, so the trigger family answers `False` rather th Every question in the introspection contract has a probe on the dialect, and a probe is a question about the server in front of you rather than about the product. This table is Snowflake's answer to every probe the families use. Read the `False` rows first: a probe that answers `False` says the question cannot be asked of this server, never that the answer was empty or zero. -Ten catalogue families answer `True` -- the ten catalogue types the CLI offers -- and so do the three data families. Two further switches answer `True`: `supports_introspection()`, the umbrella every scope is gated on, and `supports_table_storage_introspection()`, which fills one table record's `size_bytes` rather than backing a CLI type of its own. +Ten catalogue families answer `True` among the original set -- the ten catalogue types the CLI offered before the catalog-object work -- and so do the three data families. Seven more answer `True` as of this change: `routines`, `sequences`, `stages`, `file-formats`, `pipes`, `tasks` and `streams`. Two further switches answer `True`: `supports_introspection()`, the umbrella every scope is gated on, and `supports_table_storage_introspection()`, which fills one table record's `size_bytes` rather than backing a CLI type of its own. | Probe | Snowflake answers | The question it gates | |-------|-------------------|-----------------------| @@ -36,6 +39,13 @@ Ten catalogue families answer `True` -- the ten catalogue types the CLI offers - | `supports_index_introspection()` | `True` | Primary-key and unique constraints | | `supports_foreign_key_introspection()` | `True` | Foreign keys and their rules | | `supports_view_introspection()` | `True` | Views and their definitions | +| `supports_routine_introspection()` | `True` | Stored procedures and user-defined functions | +| `supports_sequence_introspection()` | `True` | Standalone sequences | +| `supports_stage_introspection()` | `True` | Named stages | +| `supports_file_format_introspection()` | `True` | Named file formats | +| `supports_pipe_introspection()` | `True` | Pipes | +| `supports_task_introspection()` | `True` | Tasks, from `SHOW TASKS` | +| `supports_stream_introspection()` | `True` | Streams, from `SHOW STREAMS` | | `supports_table_storage_introspection()` | `True` | One table's `size_bytes` slot | | `supports_table_storage_metrics()` | `True` | Billed storage, row count and clustering of one table | | `supports_column_data_profile()` | `True` | What each column of a table holds | @@ -131,6 +141,13 @@ Each of these is one catalogue read, or one `SHOW` for warehouses. None scans th | Schemas | `list_schemas()` | `INFORMATION_SCHEMA.SCHEMATA` | | Warehouses | `list_warehouses()` | `SHOW WAREHOUSES` | | Storage metrics | `get_table_storage_metrics()` | `TABLE_STORAGE_METRICS` joined to `TABLES` | +| Routines | `list_routines()` | `INFORMATION_SCHEMA.FUNCTIONS` + `PROCEDURES` | +| Sequences | `list_sequences()` | `INFORMATION_SCHEMA.SEQUENCES` | +| Stages | `list_stages()` | `INFORMATION_SCHEMA.STAGES` | +| File formats | `list_file_formats()` | `INFORMATION_SCHEMA.FILE_FORMATS` | +| Pipes | `list_pipes()` | `INFORMATION_SCHEMA.PIPES` | +| Tasks | `list_tasks()` | `SHOW TASKS` | +| Streams | `list_streams()` | `SHOW STREAMS` | ### Tables and views @@ -152,6 +169,20 @@ The composition is per concept, not per family. `CHARACTER_MAXIMUM_LENGTH` is do The other fields are the catalogue's own: `ORDINAL_POSITION`, `IS_NULLABLE` (read as `YES`, everything else is `NOT_NULL`), `COLUMN_DEFAULT`, `COMMENT`, `COLLATION_NAME`, and the raw precision/scale/length columns. +`IS_IDENTITY` answers the auto-increment question. Snowflake documents it as "Whether this column is an identity column", and the identity columns (`IDENTITY_GENERATION`, `IDENTITY_START`, `IDENTITY_INCREMENT`, `IDENTITY_ORDERED`) carry the declared numbers and ordering property. A column whose default is `some_sequence.NEXTVAL` is the second way the database can generate a value; the default text is the only place that fact appears. Both set `is_auto_increment` on the column record, and `extra["snowflake_auto_increment_source"]` says which one it was (`"identity"` or `"sequence_default"`). A column with neither is false. The identity fields are always present in `extra`, so a caller can tell "not an identity" from "identity details not read". + +`KIND = 'VIRTUAL_COLUMN'` marks a generated column, and `EXPRESSION` carries the expression that computes it. `is_generated` and `generated_expression` are filled from those two columns; a regular `COLUMN` keeps both unset, whatever text the expression slot happens to contain. + +### Routines, sequences and catalog objects + +`list_routines()` reads two views because Snowflake stores functions and procedures separately: `INFORMATION_SCHEMA.FUNCTIONS` and `INFORMATION_SCHEMA.PROCEDURES`. The columns are prefixed per view, so the parser reads the prefix off the `kind` it was given, and the two listings are merged and sorted by `(kind, name, argument_signature)`. `argument_signature` is the view's own signature text, which is how overloads are distinguished; `return_type` is `DATA_TYPE`; `language` is the view's `FUNCTION_LANGUAGE` / `PROCEDURE_LANGUAGE`. The engine-specific remainder -- volatility, security, handler, runtime, packages, integrations, secrets -- is in `extra` under the documented uppercase name. + +`list_sequences()` projects `INFORMATION_SCHEMA.SEQUENCES` onto core's `SequenceInfo`, with Snowflake's facts in `extra`. `MINIMUM_VALUE`, `MAXIMUM_VALUE` and `CYCLE_OPTION` are documented "Not applicable for Snowflake" and stay `None`; `START_VALUE` and `INCREMENT` are documented `VARCHAR` and are converted to integers. `NEXT_VALUE` is the value the sequence will produce next, not a last-produced value, so it is stored as `extra["snowflake_next_value"]` and `last_value` stays `None`. + +Stages, file formats, pipes, tasks and streams share one stable record shape (`SnowflakeCatalogObjectInfo`): name, kind, catalog, schema, owner, comment, created and last altered, with each family's engine-specific columns in `extra`. The first three are database-level Information Schema views; tasks and streams have no such view, so they are read through `SHOW TASKS` / `SHOW STREAMS`, scoped with `IN SCHEMA` when a schema is configured. `SHOW` rows are documented lowercase and are normalized before parsing. + +Snowflake identity columns are backed by an internal sequence, but that sequence is not a row in `SEQUENCES` and must not be inferred as one. Conversely, a standalone sequence used as a column default does not make the column an identity column. The two facts are deliberately separate: `list_sequences()` answers "which sequence objects exist", while `list_columns()` answers "is this column's value generated", with `extra` naming the mechanism. + ### Indexes and foreign keys Both families read constraints, because that is the structure Snowflake declares. @@ -320,7 +351,7 @@ The invocation requires `--account` and/or `--database`; connection parameters c ### Introspection Types -Nineteen types. Ten are catalogue questions, six are families this backend declines, and three are the data families. +Twenty-six types. Seventeen are catalogue questions, six are families this backend declines, and three are the data families. | Type | Description | Name | |------|-------------|------| @@ -334,6 +365,13 @@ Nineteen types. Ten are catalogue questions, six are families this backend decli | `schemas` | The schemas of `--database`, or the current one | refused | | `warehouses` | Virtual warehouses | refused | | `table-storage-metrics` | Billed storage tiers, row count and clustering | required (table) | +| `routines` | Functions and procedures: language, signature, return type, owner | refused | +| `sequences` | Standalone sequences: start, increment, next value | refused | +| `stages` | Named stages: URL, region, type | refused | +| `file-formats` | Named file formats: delimiter, format, compression | refused | +| `pipes` | Pipes: COPY definition, auto-ingest, pattern | refused | +| `tasks` | Tasks: schedule, state, definition, predecessors | refused | +| `streams` | Streams: source table, type, staleness | refused | | `unused-indexes` | Declined: `False` | refused | | `table-stats` | Declined: `False` | refused | | `runtime` | Declined: `False` | refused | @@ -409,6 +447,13 @@ python -m rhosocial.activerecord.backend.impl.snowflake introspect tables \ | `list_views(schema=None, include_system=False)` | Views | | `get_view_info(name, schema=None)` | One view's definition | | `get_table_storage_metrics(table_name, schema=None)` | Billed storage, row count, clustering | +| `list_routines(schema=None)` | Functions and procedures as one family | +| `list_sequences(schema=None)` | Standalone sequences | +| `list_stages(schema=None)` | Named stages | +| `list_file_formats(schema=None)` | Named file formats | +| `list_pipes(schema=None)` | Pipes | +| `list_tasks(schema=None)` | Tasks, from `SHOW TASKS` | +| `list_streams(schema=None)` | Streams, from `SHOW STREAMS` | | `list_column_data_profiles(table_name, schema=None)` | What each column holds | | `sample_column_values(table_name, column_name, limit=100, schema=None)` | Most frequent values | | `estimate_join_coverage(left_table, left_column, right_table, right_column, schema=None)` | Both coverage fractions | diff --git a/src/rhosocial/activerecord/backend/impl/snowflake/cli/README.md b/src/rhosocial/activerecord/backend/impl/snowflake/cli/README.md index 565b38c..4ea03e4 100644 --- a/src/rhosocial/activerecord/backend/impl/snowflake/cli/README.md +++ b/src/rhosocial/activerecord/backend/impl/snowflake/cli/README.md @@ -4,7 +4,7 @@ The `introspect` subcommand asks a live Snowflake account what it holds, and prints the answer. Every question runs through the backend's introspector, so what the CLI reports is what the library sees — the same objects, the same types, the same parsing — rather than a second implementation that can drift -from it. Nineteen types are advertised, derived from `_TYPE_CAPABILITIES` in +from it. Twenty-six types are advertised, derived from `_TYPE_CAPABILITIES` in the source rather than written out a second time, so a family cannot be answerable and unaskable at once. @@ -14,7 +14,7 @@ python -m rhosocial.activerecord.backend.impl.snowflake introspect [NAME] ## What it can ask about -Nineteen types, in the order `--help` prints them. The catalogue families +Twenty-six types, in the order `--help` prints them. The catalogue families answer True for this dialect; the operational and one-object families are offered and gated on their switch, so a run is refused (exit 2) rather than answered with an empty list. @@ -23,7 +23,7 @@ answered with an empty list. | :--- | :--- | :--- | :--- | | `tables` | — | `information_schema.tables` | Tables, with the kind the catalogue reports and their comment. | | `views` | view, optional | `information_schema.views` | Views, with their definitions; one view when named. | -| `columns` | table, required | `information_schema.columns` | One table's columns: name, type, nullability, ordinal position, default. | +| `columns` | table, required | `information_schema.columns` | One table's columns: name, type, nullability, ordinal position, default, and whether the value is identity- or sequence-generated. | | `indexes` | table, required | `information_schema.table_constraints`, `key_column_usage` | Constraints and the columns each side names. | | `foreign-keys` | table, required | `information_schema.referential_constraints` | Referential constraints. | | `database` | — | `CURRENT_DATABASE()`, `CURRENT_VERSION()` | The database this session is in and the account's version. | @@ -31,6 +31,13 @@ answered with an empty list. | `schemas` | — | `information_schema.schemata` | The schemas in a database — Snowflake's namespace level above a table. `--database` names the database to enumerate. | | `warehouses` | — | `SHOW WAREHOUSES` | Warehouses with size, state and auto-resume. This is `SHOW`, not an `information_schema` view, because no such view exists. | | `table-storage-metrics` | table, required | `information_schema.table_storage_metrics`, `information_schema.tables` | One table's billed storage: active, time travel, failsafe and clone-retained bytes, plus the row count, clustering key and kind `tables` reports. | +| `routines` | — | `information_schema.functions`, `information_schema.procedures` | Functions and stored procedures as one family: language, argument signature, return type, owner and comment. | +| `sequences` | — | `information_schema.sequences` | Standalone sequences: type, start, increment, next value, ordered property and owner. This is not the identity answer; that is on `columns`. | +| `stages` | — | `information_schema.stages` | Named stages with their URL, region and type. | +| `file-formats` | — | `information_schema.file_formats` | Named file formats with their delimiters, formats and compression. | +| `pipes` | — | `information_schema.pipes` | Pipes with the COPY definition, auto-ingest flag and pattern. | +| `tasks` | — | `SHOW TASKS` | Tasks with schedule, state, definition, predecessors and overlap policy. Scoped with `IN SCHEMA` when a schema is set. | +| `streams` | — | `SHOW STREAMS` | Streams with source table, type, mode, staleness and `stale_after`. | | `unused-indexes` | — | refused here | Offered and gated: `supports_unused_indexes_detection` answers False here, so the run is refused rather than answered empty. | | `table-stats` | — | refused here | Offered and gated on `supports_table_stats`, which answers False here. | | `runtime` | — | refused here | Offered and gated on `supports_runtime_stats`, which answers False here. | @@ -45,12 +52,14 @@ answered with an empty list. `table-storage-metrics`, `ddl`, `column-profiles`, `value-samples` and `join-coverage`. For `views` and `dependencies` it is optional: a name asks for that one view, or narrows the graph to that object's edges. The list types — -`tables`, `database`, `databases`, `schemas`, `warehouses`, +`tables`, `database`, `databases`, `schemas`, `warehouses`, `routines`, +`sequences`, `stages`, `file-formats`, `pipes`, `tasks`, `streams`, `unused-indexes`, `table-stats`, `runtime` and `index-stats` — take no name, and a stray one is refused (exit 2) rather than ignored: this backend has no -`get_*_info` method that could answer for a single database, schema or -warehouse. `schemas --database analytics` is unaffected — `--database` says -which database's schemas to enumerate, not which schema to fetch. +`get_*_info` method that could answer for a single database, schema, routine, +sequence, stage, file format, pipe, task or stream. `schemas --database +analytics` is unaffected — `--database` says which database's schemas to +enumerate, not which schema to fetch. Exit codes follow that split: **1** is a runtime failure (connection, query, unexpected) and also "the named view or table is not there"; **2** is a request @@ -136,6 +145,15 @@ python -m rhosocial.activerecord.backend.impl.snowflake introspect warehouses -o # One table's storage facts python -m rhosocial.activerecord.backend.impl.snowflake introspect table-storage-metrics orders --schema public --database analytics +# The schema objects beyond tables and views +python -m rhosocial.activerecord.backend.impl.snowflake introspect routines --schema public --database analytics +python -m rhosocial.activerecord.backend.impl.snowflake introspect sequences --schema public --database analytics +python -m rhosocial.activerecord.backend.impl.snowflake introspect stages --schema public --database analytics +python -m rhosocial.activerecord.backend.impl.snowflake introspect file-formats --schema public --database analytics +python -m rhosocial.activerecord.backend.impl.snowflake introspect pipes --schema public --database analytics +python -m rhosocial.activerecord.backend.impl.snowflake introspect tasks --schema public --database analytics +python -m rhosocial.activerecord.backend.impl.snowflake introspect streams --schema public --database analytics + # What the rows hold, not what the columns declare -- read from the rows, so exact and paid for python -m rhosocial.activerecord.backend.impl.snowflake introspect column-profiles orders python -m rhosocial.activerecord.backend.impl.snowflake introspect value-samples orders --column status --limit 50 diff --git a/src/rhosocial/activerecord/backend/impl/snowflake/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/snowflake/cli/introspect.py index e33fd07..4730151 100644 --- a/src/rhosocial/activerecord/backend/impl/snowflake/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/snowflake/cli/introspect.py @@ -54,6 +54,18 @@ "schemas": "supports_schema_introspection", "warehouses": "supports_warehouse_introspection", "table-storage-metrics": "supports_table_storage_metrics", + # ---- the schema objects Snowflake names ---- + # Core has no family for routines, sequences, stages, file formats, pipes, + # tasks or streams; each switch is declared on this dialect beside the + # formatter that answers it. `sequences` is a core `IntrospectionScope`, but + # core has no listing method, so the family is wired here like the rest. + "routines": "supports_routine_introspection", + "sequences": "supports_sequence_introspection", + "stages": "supports_stage_introspection", + "file-formats": "supports_file_format_introspection", + "pipes": "supports_pipe_introspection", + "tasks": "supports_task_introspection", + "streams": "supports_stream_introspection", # ---- the operational families ---- # Index usage, table counters, runtime facts and the dependency graph are # each a listing over the connection's scope, so none takes a NAME -- @@ -94,6 +106,16 @@ "databases", "schemas", "warehouses", + # The schema-object listings answer for the whole configured schema; no + # introspector method takes a single routine/sequence/stage name, so a + # NAME on one of them is refused rather than ignored. + "routines", + "sequences", + "stages", + "file-formats", + "pipes", + "tasks", + "streams", # The operational families describe the connection's scope rather than # one object, so a NAME is refused for the same reason as ``tables``: # ``introspect unused-indexes orders`` would read as a narrowing the @@ -285,6 +307,17 @@ def create_parser(subparsers): %(prog)s schemas --account myaccount --database mydb %(prog)s warehouses --account myaccount + # The schema objects beyond tables and views: routines (functions plus + # procedures), standalone sequences, stages, file formats, pipes, tasks + # and streams + %(prog)s routines --account myaccount --database mydb + %(prog)s sequences --account myaccount --database mydb + %(prog)s stages --account myaccount --database mydb + %(prog)s file-formats --account myaccount --database mydb + %(prog)s pipes --account myaccount --database mydb + %(prog)s tasks --account myaccount --database mydb + %(prog)s streams --account myaccount --database mydb + # Operational readings over the connection's scope: indexes no query has # used, per-table and per-index counters, live runtime facts, and the # dependency graph (which a NAME narrows to one object's edges) @@ -350,8 +383,9 @@ def create_parser(subparsers): "table-storage-metrics, ddl, column-profiles, value-samples and " "join-coverage (required); the object itself for views and " "dependencies (optional); refused for tables, database, " - "databases, schemas, warehouses, unused-indexes, table-stats, " - "index-stats and runtime" + "databases, schemas, warehouses, routines, sequences, stages, " + "file-formats, pipes, tasks, streams, unused-indexes, " + "table-stats, index-stats and runtime" ), ) # The three families that ask about a column or a second table carry the @@ -578,6 +612,47 @@ def _handle_introspect_sync(args, backend: SnowflakeBackend, provider): data = _serialize_for_output(warehouses) provider.display_results(data, title="Warehouses") + # ---- the schema objects Snowflake names -------------------------- + # Routines merges the two catalog views; sequences is core's + # ``SequenceInfo`` with Snowflake's extras; the remaining five are one + # stable catalog shape. None takes a NAME -- the listing answers for the + # configured schema. + + elif args.type == "routines": + routines = introspector.list_routines(schema=None) + data = _serialize_for_output(routines) + provider.display_results(data, title="Routines") + + elif args.type == "sequences": + sequences = introspector.list_sequences(schema=None) + data = _serialize_for_output(sequences) + provider.display_results(data, title="Sequences") + + elif args.type == "stages": + stages = introspector.list_stages(schema=None) + data = _serialize_for_output(stages) + provider.display_results(data, title="Stages") + + elif args.type == "file-formats": + file_formats = introspector.list_file_formats(schema=None) + data = _serialize_for_output(file_formats) + provider.display_results(data, title="File Formats") + + elif args.type == "pipes": + pipes = introspector.list_pipes(schema=None) + data = _serialize_for_output(pipes) + provider.display_results(data, title="Pipes") + + elif args.type == "tasks": + tasks = introspector.list_tasks(schema=None) + data = _serialize_for_output(tasks) + provider.display_results(data, title="Tasks") + + elif args.type == "streams": + streams = introspector.list_streams(schema=None) + data = _serialize_for_output(streams) + provider.display_results(data, title="Streams") + # ---- the operational families ------------------------------------- # These read a statistics collector or the catalogue rather than the # schema structure: each one is a listing of the whole connection's @@ -778,6 +853,44 @@ async def _handle_introspect_async(args, backend: AsyncSnowflakeBackend, provide data = _serialize_for_output(warehouses) provider.display_results(data, title="Warehouses") + # The async twins of the schema-object branches above: same method and + # same title, awaited. + + elif args.type == "routines": + routines = await introspector.list_routines(schema=None) + data = _serialize_for_output(routines) + provider.display_results(data, title="Routines") + + elif args.type == "sequences": + sequences = await introspector.list_sequences(schema=None) + data = _serialize_for_output(sequences) + provider.display_results(data, title="Sequences") + + elif args.type == "stages": + stages = await introspector.list_stages(schema=None) + data = _serialize_for_output(stages) + provider.display_results(data, title="Stages") + + elif args.type == "file-formats": + file_formats = await introspector.list_file_formats(schema=None) + data = _serialize_for_output(file_formats) + provider.display_results(data, title="File Formats") + + elif args.type == "pipes": + pipes = await introspector.list_pipes(schema=None) + data = _serialize_for_output(pipes) + provider.display_results(data, title="Pipes") + + elif args.type == "tasks": + tasks = await introspector.list_tasks(schema=None) + data = _serialize_for_output(tasks) + provider.display_results(data, title="Tasks") + + elif args.type == "streams": + streams = await introspector.list_streams(schema=None) + data = _serialize_for_output(streams) + provider.display_results(data, title="Streams") + # The async twins of the nine branches above: same method, same # argument and same title, awaited. They are written out rather than # shared with the sync chain because the executor is the only thing diff --git a/src/rhosocial/activerecord/backend/impl/snowflake/expression/introspection.py b/src/rhosocial/activerecord/backend/impl/snowflake/expression/introspection.py index 84f71ae..1c568d6 100644 --- a/src/rhosocial/activerecord/backend/impl/snowflake/expression/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/snowflake/expression/introspection.py @@ -47,9 +47,17 @@ __all__ = [ "SnowflakeDatabaseListQueryStatement", + "SnowflakeFileFormatListQueryStatement", + "SnowflakeFunctionListQueryStatement", + "SnowflakePipeListQueryStatement", + "SnowflakeProcedureListQueryStatement", "SnowflakeSchemaListQueryStatement", - "SnowflakeWarehouseListQueryStatement", + "SnowflakeSequenceListQueryStatement", + "SnowflakeStageListQueryStatement", + "SnowflakeStreamListQueryStatement", "SnowflakeTableStorageMetricsQueryStatement", + "SnowflakeTaskListQueryStatement", + "SnowflakeWarehouseListQueryStatement", ] @@ -211,3 +219,133 @@ def __init__( def format_method(self) -> str: """The dialect formatting method that renders this expression.""" return "format_table_storage_metrics_query_statement" + + +class SnowflakeFunctionListQueryStatement(IntrospectionExpression): + """``SELECT`` over ``INFORMATION_SCHEMA.FUNCTIONS``. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/functions + + The view lists user-defined functions, external functions and data metric + functions in the current (or specified) database. It does not list the + built-in system functions; those are served by ``SHOW FUNCTIONS`` and are + not schema objects with an owner or a comment. + """ + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_function_list_query_statement" + + +class SnowflakeProcedureListQueryStatement(IntrospectionExpression): + """``SELECT`` over ``INFORMATION_SCHEMA.PROCEDURES``. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/procedures + + Procedures are a separate catalog view from functions: the columns are + prefixed ``PROCEDURE_`` and the return-value columns differ. Keeping a + separate statement is what lets each formatter select the documented + columns instead of projecting a union of two views. + """ + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_procedure_list_query_statement" + + +class SnowflakeSequenceListQueryStatement(IntrospectionExpression): + """``SELECT`` over ``INFORMATION_SCHEMA.SEQUENCES``. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/sequences + + Snowflake sequences are standalone schema objects, like PostgreSQL's and + MariaDB's; they are not a synonym for a table's identity column, which is + answered by ``COLUMNS.IS_IDENTITY`` instead. + """ + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_sequence_list_query_statement" + + +class SnowflakeStageListQueryStatement(IntrospectionExpression): + """``SELECT`` over ``INFORMATION_SCHEMA.STAGES``. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/stages + + The view lists named stages; the user and table stages are not named + objects and do not appear here. + """ + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_stage_list_query_statement" + + +class SnowflakeFileFormatListQueryStatement(IntrospectionExpression): + """``SELECT`` over ``INFORMATION_SCHEMA.FILE_FORMATS``. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/file_formats + + The view lists named file formats, with the delimiter, encoding and + compression options each declares. + """ + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_file_format_list_query_statement" + + +class SnowflakePipeListQueryStatement(IntrospectionExpression): + """``SELECT`` over ``INFORMATION_SCHEMA.PIPES``. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/pipes + + The view lists pipes, with the COPY statement each runs and the pattern it + accepts. Current pipe status is not a catalog column; it is answered by + ``SYSTEM$PIPE_STATUS``, which is left to the caller. + """ + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_pipe_list_query_statement" + + +class SnowflakeTaskListQueryStatement(IntrospectionExpression): + """``SHOW TASKS`` -- there is no ``INFORMATION_SCHEMA.TASKS`` view. + + Documentation: https://docs.snowflake.com/en/sql-reference/sql/show-tasks + + The Information Schema view index lists no ``TASKS`` view, so the + documented route is the ``SHOW`` command, rendered by + :class:`~...expression.show.SnowflakeShowStatement`. When a schema is + given the statement is scoped to it with ``IN SCHEMA``; without one it + keeps Snowflake's own default (the current database). + """ + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_task_list_query_statement" + + +class SnowflakeStreamListQueryStatement(IntrospectionExpression): + """``SHOW STREAMS`` -- there is no ``INFORMATION_SCHEMA.STREAMS`` view. + + Documentation: https://docs.snowflake.com/en/sql-reference/sql/show-streams + + As with tasks, the documented access path is ``SHOW``. Stream rows are + documented in lowercase, and the introspector normalizes their keys before + reading them. + """ + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_stream_list_query_statement" diff --git a/src/rhosocial/activerecord/backend/impl/snowflake/introspection/__init__.py b/src/rhosocial/activerecord/backend/impl/snowflake/introspection/__init__.py index 387beea..5ef36c1 100644 --- a/src/rhosocial/activerecord/backend/impl/snowflake/introspection/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/snowflake/introspection/__init__.py @@ -11,7 +11,9 @@ AsyncSnowflakeIntrospector, ) from .types import ( + SnowflakeCatalogObjectInfo, SnowflakeDatabaseInfo, + SnowflakeRoutineInfo, SnowflakeSchemaInfo, SnowflakeTableStorageMetrics, SnowflakeWarehouseInfo, @@ -21,7 +23,9 @@ "SnowflakeIntrospectorMixin", "SnowflakeIntrospector", "AsyncSnowflakeIntrospector", + "SnowflakeCatalogObjectInfo", "SnowflakeDatabaseInfo", + "SnowflakeRoutineInfo", "SnowflakeSchemaInfo", "SnowflakeTableStorageMetrics", "SnowflakeWarehouseInfo", diff --git a/src/rhosocial/activerecord/backend/impl/snowflake/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/snowflake/introspection/introspector.py index 5d233a3..6222a78 100644 --- a/src/rhosocial/activerecord/backend/impl/snowflake/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/snowflake/introspection/introspector.py @@ -55,17 +55,28 @@ IntrospectionTriggerInfo, IntrospectionViewInfo, ReferentialAction, + SequenceInfo, ) from ..expression.introspection import ( SnowflakeDatabaseListQueryStatement, + SnowflakeFileFormatListQueryStatement, + SnowflakeFunctionListQueryStatement, + SnowflakePipeListQueryStatement, + SnowflakeProcedureListQueryStatement, SnowflakeSchemaListQueryStatement, + SnowflakeSequenceListQueryStatement, + SnowflakeStageListQueryStatement, + SnowflakeStreamListQueryStatement, SnowflakeTableStorageMetricsQueryStatement, + SnowflakeTaskListQueryStatement, SnowflakeWarehouseListQueryStatement, ) from .table_types import core_table_type, read_table_type from .types import ( + SnowflakeCatalogObjectInfo, SnowflakeDatabaseInfo, + SnowflakeRoutineInfo, SnowflakeSchemaInfo, SnowflakeTableStorageMetrics, SnowflakeWarehouseInfo, @@ -177,6 +188,41 @@ def _as_optional_flag(cls, value: Any) -> Optional[bool]: return None return cls._as_flag(value) + @staticmethod + def _as_tristate_flag(value: Any) -> Optional[bool]: + """A flag where an unreadable spelling is "not reported", not "false". + + :meth:`_as_optional_flag` maps any non-empty text to a boolean, which + is right for ``SHOW`` properties that are documented booleans: an + unreadable ``auto_resume`` is not "true". It is wrong for the catalog + columns below, where a non-flag is a spelling this backend has not + learned, and reporting it as ``False`` would invent an answer: + + * ``SEQUENCES.CYCLE_OPTION`` is documented "Not applicable for + Snowflake", so the only honest reading of an unexpected value is + "this backend does not know". + * ``SEQUENCES.ORDERED`` and ``COLUMNS.IDENTITY_ORDERED`` are + documented ``YES``/``NO``; anything else is an unlearned spelling. + * the file-format, pipe, task and stream booleans are documented as + their views/``SHOW`` pages state, and the same rule applies. + + Values that *are* recognisable (``Y``/``YES``/``TRUE``/``1`` and the + four negative spellings) answer ``True``/``False``; everything else + answers ``None``. + """ + if isinstance(value, bool): + return value + if value is None: + return None + if isinstance(value, (int, float)): + return value != 0 + text = str(value).strip().upper() + if text in ("Y", "YES", "TRUE", "1"): + return True + if text in ("N", "NO", "FALSE", "0"): + return False + return None + @staticmethod def _as_int(value: Any) -> Optional[int]: """One catalog number as an ``int``, or ``None`` when unreadable. @@ -192,6 +238,368 @@ def _as_int(value: Any) -> Optional[int]: except (TypeError, ValueError): return None + # ------------------------------------------------------------------ # + # Routines, sequences and the Snowflake catalog objects + # ------------------------------------------------------------------ # + + @staticmethod + def _is_sequence_backed(default_value: Any) -> bool: + """Whether a column default calls a sequence's ``NEXTVAL``. + + Snowflake has two ways a column can get a database-generated number. + ``IDENTITY`` / ``AUTOINCREMENT`` is one; a ``DEFAULT`` of the form + ``some_sequence.NEXTVAL`` is the other. The catalog reports the first + as ``IS_IDENTITY = 'YES'`` and the second only as default text, so + the question "is this column auto-generated?" needs both signals. + The test is deliberately narrow -- the literal word ``NEXTVAL`` in + the documented default expression -- rather than a guess at a + function name this backend has not learned. + """ + if default_value is None: + return False + return "NEXTVAL" in str(default_value).upper() + + def parse_routines( + self, + rows: List[Dict[str, Any]], + kind: str, + schema: str, + ) -> List[SnowflakeRoutineInfo]: + """Parse ``FUNCTIONS`` or ``PROCEDURES`` rows into one family shape. + + *kind* is ``"FUNCTION"`` or ``"PROCEDURE"`` and is also the prefix both + catalog views put on their columns (``FUNCTION_NAME`` / + ``PROCEDURE_NAME``). Reading the prefix off the kind is why one parser + serves both views: the documented shape is the same, the spelling is + not, and duplicate parsers would be two places for the prefix rule to + drift. + + ``additional columns`` that only one view documents are read with + ``.get`` and kept in ``extra``: the function-only volatility and + aggregate flags stay ``None`` on a procedure, and the procedure-only + ``SECRETS``/``EXTERNAL_ACCESS_INTEGRATIONS`` stay ``None`` on a + function. The extras therefore have the same keys for both kinds, + which is the stable shape a caller filters on. + """ + routines = [] + for raw_row in rows: + row = self._upper_row(raw_row) + prefix = kind.upper() + routines.append( + SnowflakeRoutineInfo( + name=row.get(f"{prefix}_NAME", ""), + kind=prefix, + schema=row.get(f"{prefix}_SCHEMA") or schema, + catalog=row.get(f"{prefix}_CATALOG"), + language=row.get(f"{prefix}_LANGUAGE"), + argument_signature=row.get("ARGUMENT_SIGNATURE"), + return_type=row.get("DATA_TYPE"), + owner=row.get(f"{prefix}_OWNER"), + comment=row.get("COMMENT"), + created=row.get("CREATED"), + last_altered=row.get("LAST_ALTERED"), + definition=row.get(f"{prefix}_DEFINITION"), + extra={ + "character_maximum_length": self._as_int( + row.get("CHARACTER_MAXIMUM_LENGTH") + ), + "character_octet_length": self._as_int( + row.get("CHARACTER_OCTET_LENGTH") + ), + "numeric_precision": self._as_int(row.get("NUMERIC_PRECISION")), + "numeric_precision_radix": self._as_int( + row.get("NUMERIC_PRECISION_RADIX") + ), + "numeric_scale": self._as_int(row.get("NUMERIC_SCALE")), + "volatility": row.get("VOLATILITY"), + "is_null_call": self._as_tristate_flag(row.get("IS_NULL_CALL")), + "is_secure": self._as_tristate_flag(row.get("IS_SECURE")), + "is_memoizable": self._as_tristate_flag(row.get("IS_MEMOIZABLE")), + "is_external": self._as_tristate_flag(row.get("IS_EXTERNAL")), + "is_data_metric": self._as_tristate_flag( + row.get("IS_DATA_METRIC") + ), + "is_aggregate": self._as_tristate_flag(row.get("IS_AGGREGATE")), + "runtime_version": row.get("RUNTIME_VERSION"), + "handler": row.get("HANDLER"), + "packages": row.get("PACKAGES"), + "installed_packages": row.get("INSTALLED_PACKAGES"), + "artifact_repository": row.get("ARTIFACT_REPOSITORY"), + "external_access_integrations": row.get( + "EXTERNAL_ACCESS_INTEGRATIONS" + ), + "secrets": row.get("SECRETS"), + "owner_role_type": row.get("OWNER_ROLE_TYPE"), + }, + ) + ) + return routines + + def parse_sequences( + self, + rows: List[Dict[str, Any]], + schema: str, + ) -> List[SequenceInfo]: + """Parse ``INFORMATION_SCHEMA.SEQUENCES`` rows onto core's ``SequenceInfo``. + + The common shape is core's, because a sequence is one of the object + kinds core already models. Snowflake's differences live in ``extra``: + the catalog's own three-part name, the owner, ``NEXT_VALUE`` (which is + *not* a last-produced value and is therefore not put in + ``last_value``), and the ``ORDERED`` property. + + ``MINIMUM_VALUE``, ``MAXIMUM_VALUE`` and ``CYCLE_OPTION`` are + documented "Not applicable for Snowflake"; they parse as ``None`` + rather than as an invented default. ``START_VALUE`` and ``INCREMENT`` + are documented ``VARCHAR`` and are converted by ``_as_int``. A value + that cannot be read costs that field its value, not the row. + """ + sequences = [] + for raw_row in rows: + row = self._upper_row(raw_row) + sequences.append( + SequenceInfo( + name=row.get("SEQUENCE_NAME", ""), + schema=row.get("SEQUENCE_SCHEMA") or schema, + data_type=row.get("DATA_TYPE"), + start_value=self._as_int(row.get("START_VALUE")), + minimum_value=self._as_int(row.get("MINIMUM_VALUE")), + maximum_value=self._as_int(row.get("MAXIMUM_VALUE")), + increment=self._as_int(row.get("INCREMENT")), + cycle=self._as_tristate_flag(row.get("CYCLE_OPTION")), + cache_size=None, + last_value=None, + extra={ + "snowflake_catalog": row.get("SEQUENCE_CATALOG"), + "snowflake_owner": row.get("SEQUENCE_OWNER"), + "snowflake_next_value": self._as_int(row.get("NEXT_VALUE")), + "snowflake_ordered": self._as_tristate_flag( + row.get("ORDERED") + ), + "snowflake_numeric_precision": self._as_int( + row.get("NUMERIC_PRECISION") + ), + "snowflake_numeric_precision_radix": self._as_int( + row.get("NUMERIC_PRECISION_RADIX") + ), + "snowflake_numeric_scale": self._as_int( + row.get("NUMERIC_SCALE") + ), + "snowflake_created": row.get("CREATED"), + "snowflake_last_altered": row.get("LAST_ALTERED"), + "snowflake_comment": row.get("COMMENT"), + }, + ) + ) + return sequences + + def parse_stages( + self, + rows: List[Dict[str, Any]], + schema: str, + ) -> List[SnowflakeCatalogObjectInfo]: + """Parse ``INFORMATION_SCHEMA.STAGES`` rows into the catalog shape.""" + stages = [] + for raw_row in rows: + row = self._upper_row(raw_row) + stages.append( + SnowflakeCatalogObjectInfo( + name=row.get("STAGE_NAME", ""), + kind="STAGE", + catalog=row.get("STAGE_CATALOG"), + schema=row.get("STAGE_SCHEMA") or schema, + owner=row.get("STAGE_OWNER"), + comment=row.get("COMMENT"), + created=row.get("CREATED"), + last_altered=row.get("LAST_ALTERED"), + extra={ + "stage_url": row.get("STAGE_URL"), + "stage_region": row.get("STAGE_REGION"), + "stage_type": row.get("STAGE_TYPE"), + }, + ) + ) + return stages + + def parse_file_formats( + self, + rows: List[Dict[str, Any]], + schema: str, + ) -> List[SnowflakeCatalogObjectInfo]: + """Parse ``INFORMATION_SCHEMA.FILE_FORMATS`` rows into the catalog shape. + + ``SKIP_HEADER`` is a ``NUMBER``, so it is converted; the boolean + options are documented as text and read through the shared flag + reader so ``TRUE``/``FALSE`` and ``YES``/``NO`` both work. + """ + formats = [] + for raw_row in rows: + row = self._upper_row(raw_row) + formats.append( + SnowflakeCatalogObjectInfo( + name=row.get("FILE_FORMAT_NAME", ""), + kind="FILE FORMAT", + catalog=row.get("FILE_FORMAT_CATALOG"), + schema=row.get("FILE_FORMAT_SCHEMA") or schema, + owner=row.get("FILE_FORMAT_OWNER"), + comment=row.get("COMMENT"), + created=row.get("CREATED"), + last_altered=row.get("LAST_ALTERED"), + extra={ + "file_format_type": row.get("FILE_FORMAT_TYPE"), + "record_delimiter": row.get("RECORD_DELIMITER"), + "field_delimiter": row.get("FIELD_DELIMITER"), + "skip_header": self._as_int(row.get("SKIP_HEADER")), + "date_format": row.get("DATE_FORMAT"), + "time_format": row.get("TIME_FORMAT"), + "timestamp_format": row.get("TIMESTAMP_FORMAT"), + "binary_format": row.get("BINARY_FORMAT"), + "escape": row.get("ESCAPE"), + "escape_unenclosed_field": row.get( + "ESCAPE_UNENCLOSED_FIELD" + ), + "trim_space": self._as_tristate_flag(row.get("TRIM_SPACE")), + "field_optionally_enclosed_by": row.get( + "FIELD_OPTIONALLY_ENCLOSED_BY" + ), + "null_if": row.get("NULL_IF"), + "compression": row.get("COMPRESSION"), + "error_on_column_count_mismatch": self._as_tristate_flag( + row.get("ERROR_ON_COLUMN_COUNT_MISMATCH") + ), + }, + ) + ) + return formats + + def parse_pipes( + self, + rows: List[Dict[str, Any]], + schema: str, + ) -> List[SnowflakeCatalogObjectInfo]: + """Parse ``INFORMATION_SCHEMA.PIPES`` rows into the catalog shape.""" + pipes = [] + for raw_row in rows: + row = self._upper_row(raw_row) + pipes.append( + SnowflakeCatalogObjectInfo( + name=row.get("PIPE_NAME", ""), + kind="PIPE", + catalog=row.get("PIPE_CATALOG"), + schema=row.get("PIPE_SCHEMA") or schema, + owner=row.get("PIPE_OWNER"), + comment=row.get("COMMENT"), + created=row.get("CREATED"), + last_altered=row.get("LAST_ALTERED"), + extra={ + "definition": row.get("DEFINITION"), + "is_autoingest_enabled": self._as_tristate_flag( + row.get("IS_AUTOINGEST_ENABLED") + ), + "notification_channel_name": row.get( + "NOTIFICATION_CHANNEL_NAME" + ), + "pattern": row.get("PATTERN"), + }, + ) + ) + return pipes + + def parse_tasks(self, rows: List[Dict[str, Any]]) -> List[SnowflakeCatalogObjectInfo]: + """Parse ``SHOW TASKS`` rows, whose column names are lowercase. + + The page documents two places where the shape is not an Information + Schema row: the keys are lowercase, and the timestamps are ``created_on`` + only -- there is no ``last_altered`` column in this output. Values are + stored as the catalog spelled them; ``ALLOW_OVERLAPPING_EXECUTION`` is + the one documented boolean and is read through the shared flag reader. + + ``PREDECESSORS`` and ``TASK_RELATIONS`` are JSON-bearing ``VARCHAR`` + fields. They are kept as text rather than parsed here: this backend + does not know whether a given account serializes them with the standard + JSON rules, and a failed parse should cost no row its identity. + """ + tasks = [] + for raw_row in rows: + row = self._upper_row(raw_row) + tasks.append( + SnowflakeCatalogObjectInfo( + name=row.get("NAME", ""), + kind="TASK", + catalog=row.get("DATABASE_NAME"), + schema=row.get("SCHEMA_NAME"), + owner=row.get("OWNER"), + comment=row.get("COMMENT"), + created=row.get("CREATED_ON"), + last_altered=None, + extra={ + "id": row.get("ID"), + "warehouse": row.get("WAREHOUSE"), + "schedule": row.get("SCHEDULE"), + "scheduling_mode": row.get("SCHEDULING_MODE"), + "target_completion_interval": row.get( + "TARGET_COMPLETION_INTERVAL" + ), + "predecessors": row.get("PREDECESSORS"), + "state": row.get("STATE"), + "definition": row.get("DEFINITION"), + "condition": row.get("CONDITION"), + "allow_overlapping_execution": self._as_tristate_flag( + row.get("ALLOW_OVERLAPPING_EXECUTION") + ), + "error_integration": row.get("ERROR_INTEGRATION"), + "success_integration": row.get("SUCCESS_INTEGRATION"), + "last_committed_on": row.get("LAST_COMMITTED_ON"), + "last_suspended_on": row.get("LAST_SUSPENDED_ON"), + "last_suspended_reason": row.get("LAST_SUSPENDED_REASON"), + "task_relations": row.get("TASK_RELATIONS"), + "execute_as_user": row.get("EXECUTE_AS_USER"), + "overlap_policy": row.get("OVERLAP_POLICY"), + "owner_role_type": row.get("OWNER_ROLE_TYPE"), + "config": row.get("CONFIG"), + }, + ) + ) + return tasks + + def parse_streams( + self, rows: List[Dict[str, Any]] + ) -> List[SnowflakeCatalogObjectInfo]: + """Parse ``SHOW STREAMS`` rows, whose column names are lowercase. + + ``STALE`` is the documented boolean ("If this is TRUE, the stream may + be stale"); it and ``STALE_AFTER`` are the two columns a consumer acts + on. ``BASE_TABLES`` applies only to streams on views and is kept as the + catalog text. There is no ``last_altered`` in this output. + """ + streams = [] + for raw_row in rows: + row = self._upper_row(raw_row) + streams.append( + SnowflakeCatalogObjectInfo( + name=row.get("NAME", ""), + kind="STREAM", + catalog=row.get("DATABASE_NAME"), + schema=row.get("SCHEMA_NAME"), + owner=row.get("OWNER"), + comment=row.get("COMMENT"), + created=row.get("CREATED_ON"), + last_altered=None, + extra={ + "table_name": row.get("TABLE_NAME"), + "source_type": row.get("SOURCE_TYPE"), + "base_tables": row.get("BASE_TABLES"), + "stream_type": row.get("TYPE"), + "stale": self._as_tristate_flag(row.get("STALE")), + "mode": row.get("MODE"), + "stale_after": row.get("STALE_AFTER"), + "invalid_reason": row.get("INVALID_REASON"), + "owner_role_type": row.get("OWNER_ROLE_TYPE"), + }, + ) + ) + return streams + def parse_databases(self, rows: List[Dict[str, Any]]) -> List[SnowflakeDatabaseInfo]: """Parse ``INFORMATION_SCHEMA.DATABASES`` rows. @@ -671,6 +1079,17 @@ def parse_columns( else ColumnNullable.NOT_NULL ) data_type = row.get("DATA_TYPE") or "VARCHAR" + # Identity and sequence-backed defaults are different spellings of + # the same first-order fact: the database supplies the value. Both + # set ``is_auto_increment``; ``extra`` keeps which one it was, so + # a model builder can tell an AUTOINCREMENT column from one that + # merely defaults to ``sequence.NEXTVAL``. + identity = str(row.get("IS_IDENTITY") or "").strip().upper() == "YES" + sequence_backed = self._is_sequence_backed(row.get("COLUMN_DEFAULT")) + kind = str(row.get("KIND") or "").strip().upper() + generated_expression = ( + row.get("EXPRESSION") if kind == "VIRTUAL_COLUMN" else None + ) # ``data_type`` stays the bare word the catalog reported and # ``data_type_full`` carries the size, which is what the two fields # are for: the string core's differ falls back to when a type cannot @@ -695,6 +1114,33 @@ def parse_columns( numeric_precision=row.get("NUMERIC_PRECISION"), numeric_scale=row.get("NUMERIC_SCALE"), collation=row.get("COLLATION_NAME"), + is_auto_increment=identity or sequence_backed, + is_generated=kind == "VIRTUAL_COLUMN", + generated_expression=generated_expression, + extra={ + "snowflake_auto_increment_source": ( + "identity" + if identity + else ("sequence_default" if sequence_backed else None) + ), + "snowflake_identity": identity, + "snowflake_identity_generation": row.get( + "IDENTITY_GENERATION" + ), + "snowflake_identity_start": self._as_int( + row.get("IDENTITY_START") + ), + "snowflake_identity_increment": self._as_int( + row.get("IDENTITY_INCREMENT") + ), + "snowflake_identity_ordered": self._as_tristate_flag( + row.get("IDENTITY_ORDERED") + ), + "snowflake_sequence_backed": sequence_backed, + "snowflake_column_kind": row.get("KIND"), + "snowflake_data_type_alias": row.get("DATA_TYPE_ALIAS"), + "snowflake_expression": row.get("EXPRESSION"), + }, ) ) return columns @@ -1059,6 +1505,199 @@ def get_table_storage_metrics( self._cache.set(key, result) return result + # ------------------------------------------------------------------ # + # Routines, sequences and the Snowflake catalog objects + # + # Method names and signatures match the async class exactly; see the + # async twins below. Each answer is cached under ``IntrospectionScope.SCHEMA`` + # with the family in ``extra``, because core has no scope for a routine, + # a stage or a stream -- and the ``extra`` is what keeps five families + # that share the scope from answering each other's questions. + # ------------------------------------------------------------------ # + + def list_routines(self, schema: Optional[str] = None) -> List[SnowflakeRoutineInfo]: + """List stored procedures and user-defined functions of one schema. + + Two statements are read -- ``INFORMATION_SCHEMA.FUNCTIONS`` and + ``INFORMATION_SCHEMA.PROCEDURES`` -- because the engine keeps them + in two views with different columns. The two answers are merged and + sorted by ``(kind, name, argument_signature)`` so overloads have a + deterministic order. The function and procedure columns are + documented on the pages linked in + :meth:`~..mixins.introspection.SnowflakeIntrospectionMixin.format_function_list_query_statement` + and + :meth:`~..mixins.introspection.SnowflakeIntrospectionMixin.format_procedure_list_query_statement`. + + ``schema=None`` means the configured schema, as everywhere else. A + result is cached per schema; the cache key carries the family because + core has no routine scope. + + The view is privilege-filtered: a role that cannot see a routine does + not get a row for it, which is not the same as the routine not existing. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key(IntrospectionScope.SCHEMA, schema=target_schema, extra="routines") + cached = self._cache.get(key) + if cached is not None: + return cached + + functions_sql, functions_params = SnowflakeFunctionListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + functions = self.parse_routines( + self._executor.execute(functions_sql, functions_params), + "FUNCTION", + target_schema, + ) + procedures_sql, procedures_params = SnowflakeProcedureListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + procedures = self.parse_routines( + self._executor.execute(procedures_sql, procedures_params), + "PROCEDURE", + target_schema, + ) + result = sorted( + functions + procedures, + key=lambda routine: ( + routine.kind, + routine.name, + routine.argument_signature or "", + ), + ) + self._cache.set(key, result) + return result + + def list_sequences(self, schema: Optional[str] = None) -> List[SequenceInfo]: + """List standalone sequences of one schema. + + Reads ``INFORMATION_SCHEMA.SEQUENCES``; the columns and the meaning + of each mapped field are in + :meth:`~..mixins.introspection.SnowflakeIntrospectorMixin.parse_sequences`. + + This family is not the identity answer. A column declared + ``AUTOINCREMENT`` has no independent sequence object in + ``SEQUENCES`` (Snowflake manages its backing sequence internally), and + ``COLUMNS.IS_IDENTITY`` is where that fact is read; see + :meth:`~..mixins.introspection.SnowflakeIntrospectorMixin.parse_columns`. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SEQUENCE, schema=target_schema, extra="sequences" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeSequenceListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_sequences( + self._executor.execute(sql, params), target_schema + ) + self._cache.set(key, result) + return result + + def list_stages(self, schema: Optional[str] = None) -> List[SnowflakeCatalogObjectInfo]: + """List named stages of one schema, from ``INFORMATION_SCHEMA.STAGES``.""" + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="stages" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeStageListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_stages(self._executor.execute(sql, params), target_schema) + self._cache.set(key, result) + return result + + def list_file_formats( + self, schema: Optional[str] = None + ) -> List[SnowflakeCatalogObjectInfo]: + """List named file formats of one schema, from ``INFORMATION_SCHEMA.FILE_FORMATS``.""" + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="file_formats" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeFileFormatListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_file_formats( + self._executor.execute(sql, params), target_schema + ) + self._cache.set(key, result) + return result + + def list_pipes(self, schema: Optional[str] = None) -> List[SnowflakeCatalogObjectInfo]: + """List pipes of one schema, from ``INFORMATION_SCHEMA.PIPES``.""" + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="pipes" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakePipeListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_pipes(self._executor.execute(sql, params), target_schema) + self._cache.set(key, result) + return result + + def list_tasks(self, schema: Optional[str] = None) -> List[SnowflakeCatalogObjectInfo]: + """List tasks of one schema, from ``SHOW TASKS``. + + There is no Information Schema view for tasks; the statement is scoped + with ``IN SCHEMA`` when a schema is known and is the bare command when + it is not. The rows are documented lowercase and are normalized before + parsing. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="tasks" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeTaskListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_tasks(self._executor.execute(sql, params)) + self._cache.set(key, result) + return result + + def list_streams(self, schema: Optional[str] = None) -> List[SnowflakeCatalogObjectInfo]: + """List streams of one schema, from ``SHOW STREAMS``. + + The same absence of a view as for tasks, and the same schema scope rule + and lowercase row shape. ``STALE``/``STALE_AFTER`` say whether the + stream can still be read. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="streams" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeStreamListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_streams(self._executor.execute(sql, params)) + self._cache.set(key, result) + return result + class _SnowflakeAsyncIntrospectorExecutor: """Async executor wrapping synchronous Snowflake cursor operations. @@ -1213,3 +1852,184 @@ async def get_table_storage_metrics( if result is not None: self._cache.set(key, result) return result + + # ------------------------------------------------------------------ # + # Routines, sequences and the Snowflake catalog objects + # Async twins of the sync methods above: same method names, same + # signatures, same cache keys, awaited execution. + # ------------------------------------------------------------------ # + + async def list_routines( + self, schema: Optional[str] = None + ) -> List[SnowflakeRoutineInfo]: + """List stored procedures and user-defined functions asynchronously. + + Asynchronous twin of :meth:`SnowflakeIntrospector.list_routines`; both + catalog views are read, merged and sorted the same way. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key(IntrospectionScope.SCHEMA, schema=target_schema, extra="routines") + cached = self._cache.get(key) + if cached is not None: + return cached + + functions_sql, functions_params = SnowflakeFunctionListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + functions = self.parse_routines( + await self._executor.execute(functions_sql, functions_params), + "FUNCTION", + target_schema, + ) + procedures_sql, procedures_params = SnowflakeProcedureListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + procedures = self.parse_routines( + await self._executor.execute(procedures_sql, procedures_params), + "PROCEDURE", + target_schema, + ) + result = sorted( + functions + procedures, + key=lambda routine: ( + routine.kind, + routine.name, + routine.argument_signature or "", + ), + ) + self._cache.set(key, result) + return result + + async def list_sequences(self, schema: Optional[str] = None) -> List[SequenceInfo]: + """List standalone sequences asynchronously. + + Asynchronous twin of :meth:`SnowflakeIntrospector.list_sequences`. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SEQUENCE, schema=target_schema, extra="sequences" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeSequenceListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_sequences( + await self._executor.execute(sql, params), target_schema + ) + self._cache.set(key, result) + return result + + async def list_stages( + self, schema: Optional[str] = None + ) -> List[SnowflakeCatalogObjectInfo]: + """List named stages asynchronously. + + Asynchronous twin of :meth:`SnowflakeIntrospector.list_stages`. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="stages" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeStageListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_stages(await self._executor.execute(sql, params), target_schema) + self._cache.set(key, result) + return result + + async def list_file_formats( + self, schema: Optional[str] = None + ) -> List[SnowflakeCatalogObjectInfo]: + """List named file formats asynchronously. + + Asynchronous twin of :meth:`SnowflakeIntrospector.list_file_formats`. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="file_formats" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeFileFormatListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_file_formats( + await self._executor.execute(sql, params), target_schema + ) + self._cache.set(key, result) + return result + + async def list_pipes( + self, schema: Optional[str] = None + ) -> List[SnowflakeCatalogObjectInfo]: + """List pipes asynchronously. + + Asynchronous twin of :meth:`SnowflakeIntrospector.list_pipes`. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="pipes" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakePipeListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_pipes(await self._executor.execute(sql, params), target_schema) + self._cache.set(key, result) + return result + + async def list_tasks( + self, schema: Optional[str] = None + ) -> List[SnowflakeCatalogObjectInfo]: + """List tasks asynchronously. + + Asynchronous twin of :meth:`SnowflakeIntrospector.list_tasks`. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="tasks" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeTaskListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_tasks(await self._executor.execute(sql, params)) + self._cache.set(key, result) + return result + + async def list_streams( + self, schema: Optional[str] = None + ) -> List[SnowflakeCatalogObjectInfo]: + """List streams asynchronously. + + Asynchronous twin of :meth:`SnowflakeIntrospector.list_streams`. + """ + target_schema = schema if schema is not None else self.default_schema + key = self._cache.key( + IntrospectionScope.SCHEMA, schema=target_schema, extra="streams" + ) + cached = self._cache.get(key) + if cached is not None: + return cached + + sql, params = SnowflakeStreamListQueryStatement( + self.dialect, schema=target_schema + ).to_sql() + result = self.parse_streams(await self._executor.execute(sql, params)) + self._cache.set(key, result) + return result diff --git a/src/rhosocial/activerecord/backend/impl/snowflake/introspection/types.py b/src/rhosocial/activerecord/backend/impl/snowflake/introspection/types.py index af92dee..ecce599 100644 --- a/src/rhosocial/activerecord/backend/impl/snowflake/introspection/types.py +++ b/src/rhosocial/activerecord/backend/impl/snowflake/introspection/types.py @@ -19,9 +19,9 @@ class here mirrors one documented Snowflake result set and says which the parser and the SQL text but cannot pin the account's answer. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional @dataclass(frozen=True) @@ -226,3 +226,78 @@ def is_dropped(self) -> bool: def is_clustered(self) -> bool: """Whether ``TABLES.CLUSTERING_KEY`` names a clustering key.""" return bool(self.clustering_key) + + +@dataclass(frozen=True) +class SnowflakeRoutineInfo: + """One stored procedure or user-defined function, as Snowflake reports it. + + ``kind`` is ``"FUNCTION"`` or ``"PROCEDURE"``. Both catalog views carry + their own prefix on every column (``FUNCTION_NAME`` / ``PROCEDURE_NAME``); + the parser reads the prefix off *kind* rather than keeping two nearly + identical classes, which is the same split the engine itself makes. + + The stable slots are the ones a model inference needs: what the routine is + called, what language it is written in, the argument signature it accepts, + the type it returns, who owns it and the comment left for a reader. Each + remaining catalog fact -- volatility, security, handler, runtime, packages, + external integrations, secrets -- is engine-specific and lives in + ``extra``. The views are documented on + https://docs.snowflake.com/en/sql-reference/info-schema/functions and + https://docs.snowflake.com/en/sql-reference/info-schema/procedures. + + ``argument_signature`` is the view's own spelling (for example + ``(VARCHAR, NUMBER)``), not a re-rendered Python signature: it is the text + Snowflake uses to distinguish overloads, and rewriting it would lose that + identity. + + ``return_type`` is ``DATA_TYPE`` as the catalog reports it. The procedure + view documents it as the return value's type; a procedure can return a + single value or a table, so the field is the view's answer rather than a + shape this backend infers. + """ + + name: str + kind: str + schema: Optional[str] = None + catalog: Optional[str] = None + language: Optional[str] = None + argument_signature: Optional[str] = None + return_type: Optional[str] = None + owner: Optional[str] = None + comment: Optional[str] = None + created: Optional[datetime] = None + last_altered: Optional[datetime] = None + definition: Optional[str] = None + extra: Dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class SnowflakeCatalogObjectInfo: + """One schema object Snowflake names, as one stable shape. + + Stages, file formats and pipes are database-level views; tasks and streams + are ``SHOW`` result sets. Their column names differ in every family, but + every row answers the same first-order questions -- its name, its kind, + where it lives, who owns it, when it was created and altered, and the + comment left for a reader -- so this is the one shape those five families + project onto. ``kind`` is the engine's vocabulary (``STAGE``, + ``FILE FORMAT``, ``PIPE``, ``TASK``, ``STREAM``), not a Python type. + + Each family's remaining catalog columns are in ``extra``: a stage's URL + and region, a file format's delimiters and compression, a pipe's COPY + definition and pattern, a task's schedule and state, a stream's source + table and staleness. Keeping those fields out of the common shape is what + lets the five families share one class without pretending a stage and a + stream declare the same things. + """ + + name: str + kind: str + catalog: Optional[str] = None + schema: Optional[str] = None + owner: Optional[str] = None + comment: Optional[str] = None + created: Optional[datetime] = None + last_altered: Optional[datetime] = None + extra: Dict[str, Any] = field(default_factory=dict) diff --git a/src/rhosocial/activerecord/backend/impl/snowflake/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/snowflake/mixins/introspection.py index d39e645..38b99df 100644 --- a/src/rhosocial/activerecord/backend/impl/snowflake/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/snowflake/mixins/introspection.py @@ -35,8 +35,16 @@ from ..expression.introspection import ( SnowflakeDatabaseListQueryStatement, + SnowflakeFileFormatListQueryStatement, + SnowflakeFunctionListQueryStatement, + SnowflakePipeListQueryStatement, + SnowflakeProcedureListQueryStatement, SnowflakeSchemaListQueryStatement, + SnowflakeSequenceListQueryStatement, + SnowflakeStageListQueryStatement, + SnowflakeStreamListQueryStatement, SnowflakeTableStorageMetricsQueryStatement, + SnowflakeTaskListQueryStatement, SnowflakeWarehouseListQueryStatement, ) from ..expression.show import SnowflakeShowObjectType, SnowflakeShowStatement @@ -147,6 +155,68 @@ def supports_table_storage_introspection(self) -> bool: """ return True + # ------------------------------------------------------------------ + # Schema-object families core does not model yet + # ------------------------------------------------------------------ + # Core's ``IntrospectionSupport`` protocol names the families every + # backend shares; routines and the Snowflake catalog objects below have + # no core member, so these switches are declared here, beside the + # formatters that answer for them. The CLI's ``_TYPE_CAPABILITIES`` + # names each switch, and the tests refuse a type whose switch the + # dialect does not have -- so a family cannot be offered without a + # documented answer behind it. + + def supports_routine_introspection(self) -> bool: + """Snowflake answers for stored procedures and user-defined functions. + + The answer is a read of ``INFORMATION_SCHEMA.PROCEDURES`` and + ``INFORMATION_SCHEMA.FUNCTIONS``, joined in the introspector into one + ``routines`` family so a caller asking "what executes here" does not + have to know which of the two objects it is asking about. + """ + return True + + def supports_sequence_introspection(self) -> bool: + """Snowflake answers for standalone sequences. + + ``INFORMATION_SCHEMA.SEQUENCES`` lists them with their type, start + value, increment, next value and owner. This is not the identity + family: a column declared ``AUTOINCREMENT`` is reported by + ``COLUMNS.IS_IDENTITY``, while a standalone sequence is an object of + its own and can also be used as a column default. + """ + return True + + def supports_stage_introspection(self) -> bool: + """Snowflake answers for named stages from ``INFORMATION_SCHEMA.STAGES``.""" + return True + + def supports_file_format_introspection(self) -> bool: + """Snowflake answers for named file formats from ``INFORMATION_SCHEMA.FILE_FORMATS``.""" + return True + + def supports_pipe_introspection(self) -> bool: + """Snowflake answers for pipes from ``INFORMATION_SCHEMA.PIPES``.""" + return True + + def supports_task_introspection(self) -> bool: + """Snowflake answers for tasks, from ``SHOW TASKS``. + + No Information Schema view exists for tasks, so the documented route + is the ``SHOW`` command. Its rows are lowercase and it accepts an + ``IN SCHEMA`` scope, both of which the statement and parser handle. + """ + return True + + def supports_stream_introspection(self) -> bool: + """Snowflake answers for streams, from ``SHOW STREAMS``. + + As with tasks, there is no Information Schema view; ``SHOW STREAMS`` + is the documented route, scoped to the configured schema when one is + known. + """ + return True + # ========== Capability Detection ========== def supports_introspection(self) -> bool: @@ -192,6 +262,7 @@ def get_supported_introspection_scopes(self) -> List["IntrospectionScope"]: IntrospectionScope.INDEX, IntrospectionScope.FOREIGN_KEY, IntrospectionScope.VIEW, + IntrospectionScope.SEQUENCE, ] # ========== The catalog read, expressed once ========== @@ -430,7 +501,18 @@ def format_table_list_query_statement( def format_column_info_query_statement( self, expr: "ColumnInfoQueryStatement" ) -> Tuple[str, tuple]: - """Format column information query using INFORMATION_SCHEMA.COLUMNS.""" + """Format column information query using INFORMATION_SCHEMA.COLUMNS. + + The identity columns are selected because they answer a real question + this backend reports on: whether a column's value is generated by the + database. Snowflake documents ``IS_IDENTITY`` as "Whether this column + is an identity column", ``IDENTITY_GENERATION`` as always ``BY DEFAULT``, + ``IDENTITY_START``/``IDENTITY_INCREMENT`` as the declared numbers, and + ``IDENTITY_ORDERED`` as the sequence-order property. ``KIND`` and + ``EXPRESSION`` describe a virtual column, which is a generated column + in this engine's vocabulary. + https://docs.snowflake.com/en/sql-reference/info-schema/columns + """ params = expr.get_params() table_name = params.get("table", params.get("table_name", "")) schema = params.get("schema", "") @@ -442,11 +524,19 @@ def format_column_info_query_statement( self._catalog_column("COLUMN_DEFAULT"), self._catalog_column("IS_NULLABLE"), self._catalog_column("DATA_TYPE"), + self._catalog_column("DATA_TYPE_ALIAS"), self._catalog_column("CHARACTER_MAXIMUM_LENGTH"), self._catalog_column("NUMERIC_PRECISION"), self._catalog_column("NUMERIC_SCALE"), self._catalog_column("COLLATION_NAME"), self._catalog_column("COMMENT"), + self._catalog_column("IS_IDENTITY"), + self._catalog_column("IDENTITY_GENERATION"), + self._catalog_column("IDENTITY_START"), + self._catalog_column("IDENTITY_INCREMENT"), + self._catalog_column("IDENTITY_ORDERED"), + self._catalog_column("KIND"), + self._catalog_column("EXPRESSION"), ], self._information_schema_view("COLUMNS"), [ @@ -1376,3 +1466,380 @@ def format_table_storage_metrics_query_statement( # current-version filter is ever widened. [(self._catalog_column("TABLE_CREATED", "m"), "DESC")], ) + + # ------------------------------------------------------------------ # + # Routines, sequences and Snowflake's own catalog objects + # + # Each formatter below selects the columns documented on the linked page + # and filters on the schema the caller configured. The schema is a bind + # value, not an identifier: it is compared to a catalog column, and an + # unquoted object is stored uppercase, so a caller passes the catalog's + # spelling. None of these statements selects ``*``: the Snowflake + # Information Schema topic asks readers not to ("The Snowflake-specific + # views are subject to change... select the columns that you want"). + # ------------------------------------------------------------------ # + + def format_function_list_query_statement( + self, expr: SnowflakeFunctionListQueryStatement + ) -> Tuple[str, tuple]: + """``INFORMATION_SCHEMA.FUNCTIONS``, one row per user-defined function. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/functions + + The view lists UDFs, external functions and data metric functions -- it + does **not** list Snowflake's built-in system functions, which are not + schema objects with an owner or a comment. The columns selected are the + documented ones that answer the routine family's stable shape plus the + engine-specific remainder stored in ``extra``: + + * ``FUNCTION_NAME``, ``FUNCTION_SCHEMA``, ``FUNCTION_CATALOG`` -- + identity and place. + * ``FUNCTION_OWNER``, ``OWNER_ROLE_TYPE`` -- ownership. + * ``ARGUMENT_SIGNATURE`` -- the view's own signature, which is how + overloads are distinguished. + * ``DATA_TYPE`` -- the return type. + * ``FUNCTION_LANGUAGE``, ``RUNTIME_VERSION``, ``HANDLER`` -- + what runs the body. + * ``VOLATILITY``, ``IS_NULL_CALL``, ``IS_SECURE``, ``IS_MEMOIZABLE``, + ``IS_EXTERNAL``, ``IS_DATA_METRIC``, ``IS_AGGREGATE`` -- declared + behaviour. + * ``PACKAGES``, ``INSTALLED_PACKAGES``, ``ARTIFACT_REPOSITORY``, + ``EXTERNAL_ACCESS_INTEGRATIONS``, ``SECRETS`` -- runtime + dependencies. + * ``CREATED``, ``LAST_ALTERED``, ``COMMENT``, ``FUNCTION_DEFINITION``. + + The view is a database-level one, so the filter is on + ``FUNCTION_SCHEMA``; ``ORDER BY FUNCTION_NAME, ARGUMENT_SIGNATURE`` + makes overloads deterministic. + """ + return self._catalog_query( + [ + self._catalog_column("FUNCTION_CATALOG"), + self._catalog_column("FUNCTION_SCHEMA"), + self._catalog_column("FUNCTION_NAME"), + self._catalog_column("FUNCTION_OWNER"), + self._catalog_column("ARGUMENT_SIGNATURE"), + self._catalog_column("DATA_TYPE"), + self._catalog_column("CHARACTER_MAXIMUM_LENGTH"), + self._catalog_column("CHARACTER_OCTET_LENGTH"), + self._catalog_column("NUMERIC_PRECISION"), + self._catalog_column("NUMERIC_PRECISION_RADIX"), + self._catalog_column("NUMERIC_SCALE"), + self._catalog_column("FUNCTION_LANGUAGE"), + self._catalog_column("FUNCTION_DEFINITION"), + self._catalog_column("VOLATILITY"), + self._catalog_column("IS_NULL_CALL"), + self._catalog_column("IS_SECURE"), + self._catalog_column("IS_MEMOIZABLE"), + self._catalog_column("IS_EXTERNAL"), + self._catalog_column("IS_DATA_METRIC"), + self._catalog_column("IS_AGGREGATE"), + self._catalog_column("RUNTIME_VERSION"), + self._catalog_column("HANDLER"), + self._catalog_column("PACKAGES"), + self._catalog_column("INSTALLED_PACKAGES"), + self._catalog_column("ARTIFACT_REPOSITORY"), + self._catalog_column("EXTERNAL_ACCESS_INTEGRATIONS"), + self._catalog_column("SECRETS"), + self._catalog_column("CREATED"), + self._catalog_column("LAST_ALTERED"), + self._catalog_column("COMMENT"), + self._catalog_column("OWNER_ROLE_TYPE"), + ], + self._information_schema_view("FUNCTIONS"), + [ + ComparisonPredicate( + self, + "eq", + self._catalog_column("FUNCTION_SCHEMA"), + Literal(self, expr.schema or ""), + ) + ], + [ + self._catalog_column("FUNCTION_NAME"), + self._catalog_column("ARGUMENT_SIGNATURE"), + ], + ) + + def format_procedure_list_query_statement( + self, expr: SnowflakeProcedureListQueryStatement + ) -> Tuple[str, tuple]: + """``INFORMATION_SCHEMA.PROCEDURES``, one row per stored procedure. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/procedures + + The procedure view differs from the function view in its column prefix + and in the runtime facts it documents: ``PROCEDURES`` has no + ``VOLATILITY``, ``IS_MEMOIZABLE``, ``IS_AGGREGATE`` or ``HANDLER`` + column, and it adds ``EXTERNAL_ACCESS_INTEGRATIONS``, ``SECRETS``, + ``RUNTIME_VERSION``, ``PACKAGES``, ``INSTALLED_PACKAGES`` and + ``ARTIFACT_REPOSITORY``. Selecting the union of both views would + claim columns the view does not have, so each formatter names its + own page's columns. + """ + return self._catalog_query( + [ + self._catalog_column("PROCEDURE_CATALOG"), + self._catalog_column("PROCEDURE_SCHEMA"), + self._catalog_column("PROCEDURE_NAME"), + self._catalog_column("PROCEDURE_OWNER"), + self._catalog_column("ARGUMENT_SIGNATURE"), + self._catalog_column("DATA_TYPE"), + self._catalog_column("CHARACTER_MAXIMUM_LENGTH"), + self._catalog_column("CHARACTER_OCTET_LENGTH"), + self._catalog_column("NUMERIC_PRECISION"), + self._catalog_column("NUMERIC_PRECISION_RADIX"), + self._catalog_column("NUMERIC_SCALE"), + self._catalog_column("PROCEDURE_LANGUAGE"), + self._catalog_column("PROCEDURE_DEFINITION"), + self._catalog_column("RUNTIME_VERSION"), + self._catalog_column("PACKAGES"), + self._catalog_column("INSTALLED_PACKAGES"), + self._catalog_column("ARTIFACT_REPOSITORY"), + self._catalog_column("EXTERNAL_ACCESS_INTEGRATIONS"), + self._catalog_column("SECRETS"), + self._catalog_column("CREATED"), + self._catalog_column("LAST_ALTERED"), + self._catalog_column("COMMENT"), + self._catalog_column("OWNER_ROLE_TYPE"), + ], + self._information_schema_view("PROCEDURES"), + [ + ComparisonPredicate( + self, + "eq", + self._catalog_column("PROCEDURE_SCHEMA"), + Literal(self, expr.schema or ""), + ) + ], + [ + self._catalog_column("PROCEDURE_NAME"), + self._catalog_column("ARGUMENT_SIGNATURE"), + ], + ) + + def format_sequence_list_query_statement( + self, expr: SnowflakeSequenceListQueryStatement + ) -> Tuple[str, tuple]: + """``INFORMATION_SCHEMA.SEQUENCES``, one row per standalone sequence. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/sequences + + ``MINIMUM_VALUE``, ``MAXIMUM_VALUE`` and ``CYCLE_OPTION`` are documented + "Not applicable for Snowflake", and ``START_VALUE`` / ``INCREMENT`` / + ``NEXT_VALUE`` are ``VARCHAR`` rather than numbers. The parser reads + those three as absent and converts the string-valued numbers; the + projection keeps what the catalog reports rather than substituting + invented limits. + + The view has no "last value produced" column: ``NEXT_VALUE`` is the + value the sequence will produce next, which is a different fact and + is stored under that name in ``extra`` instead of being mislabelled + ``last_value``. + """ + return self._catalog_query( + [ + self._catalog_column("SEQUENCE_CATALOG"), + self._catalog_column("SEQUENCE_SCHEMA"), + self._catalog_column("SEQUENCE_NAME"), + self._catalog_column("SEQUENCE_OWNER"), + self._catalog_column("DATA_TYPE"), + self._catalog_column("NUMERIC_PRECISION"), + self._catalog_column("NUMERIC_PRECISION_RADIX"), + self._catalog_column("NUMERIC_SCALE"), + self._catalog_column("START_VALUE"), + self._catalog_column("MINIMUM_VALUE"), + self._catalog_column("MAXIMUM_VALUE"), + self._catalog_column("NEXT_VALUE"), + self._catalog_column("INCREMENT"), + self._catalog_column("CYCLE_OPTION"), + self._catalog_column("CREATED"), + self._catalog_column("LAST_ALTERED"), + self._catalog_column("ORDERED"), + self._catalog_column("COMMENT"), + ], + self._information_schema_view("SEQUENCES"), + [ + ComparisonPredicate( + self, + "eq", + self._catalog_column("SEQUENCE_SCHEMA"), + Literal(self, expr.schema or ""), + ) + ], + [self._catalog_column("SEQUENCE_NAME")], + ) + + def format_stage_list_query_statement( + self, expr: SnowflakeStageListQueryStatement + ) -> Tuple[str, tuple]: + """``INFORMATION_SCHEMA.STAGES``, one row per named stage. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/stages + + Only named stages appear in this view; the user stage ``~`` and table + stages are addressed by their table and are not objects with an owner + or a catalog row. + """ + return self._catalog_query( + [ + self._catalog_column("STAGE_CATALOG"), + self._catalog_column("STAGE_SCHEMA"), + self._catalog_column("STAGE_NAME"), + self._catalog_column("STAGE_URL"), + self._catalog_column("STAGE_REGION"), + self._catalog_column("STAGE_TYPE"), + self._catalog_column("STAGE_OWNER"), + self._catalog_column("COMMENT"), + self._catalog_column("CREATED"), + self._catalog_column("LAST_ALTERED"), + ], + self._information_schema_view("STAGES"), + [ + ComparisonPredicate( + self, + "eq", + self._catalog_column("STAGE_SCHEMA"), + Literal(self, expr.schema or ""), + ) + ], + [self._catalog_column("STAGE_NAME")], + ) + + def format_file_format_list_query_statement( + self, expr: SnowflakeFileFormatListQueryStatement + ) -> Tuple[str, tuple]: + """``INFORMATION_SCHEMA.FILE_FORMATS``, one row per named file format. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/file_formats + + The view documents the delimiter, encoding and compression options + explicitly, so all of them are selected: they are the facts a caller + asking "what does this format declare" wants. The two boolean + spellings in this view are catalog text (``TRUE``/``FALSE`` and + ``YES``/``NO`` depending on the column), so the parser reads them + through the shared flag reader rather than assuming one spelling. + """ + return self._catalog_query( + [ + self._catalog_column("FILE_FORMAT_CATALOG"), + self._catalog_column("FILE_FORMAT_SCHEMA"), + self._catalog_column("FILE_FORMAT_NAME"), + self._catalog_column("FILE_FORMAT_OWNER"), + self._catalog_column("FILE_FORMAT_TYPE"), + self._catalog_column("RECORD_DELIMITER"), + self._catalog_column("FIELD_DELIMITER"), + self._catalog_column("SKIP_HEADER"), + self._catalog_column("DATE_FORMAT"), + self._catalog_column("TIME_FORMAT"), + self._catalog_column("TIMESTAMP_FORMAT"), + self._catalog_column("BINARY_FORMAT"), + self._catalog_column("ESCAPE"), + self._catalog_column("ESCAPE_UNENCLOSED_FIELD"), + self._catalog_column("TRIM_SPACE"), + self._catalog_column("FIELD_OPTIONALLY_ENCLOSED_BY"), + self._catalog_column("NULL_IF"), + self._catalog_column("COMPRESSION"), + self._catalog_column("ERROR_ON_COLUMN_COUNT_MISMATCH"), + self._catalog_column("CREATED"), + self._catalog_column("LAST_ALTERED"), + self._catalog_column("COMMENT"), + ], + self._information_schema_view("FILE_FORMATS"), + [ + ComparisonPredicate( + self, + "eq", + self._catalog_column("FILE_FORMAT_SCHEMA"), + Literal(self, expr.schema or ""), + ) + ], + [self._catalog_column("FILE_FORMAT_NAME")], + ) + + def format_pipe_list_query_statement( + self, expr: SnowflakePipeListQueryStatement + ) -> Tuple[str, tuple]: + """``INFORMATION_SCHEMA.PIPES``, one row per pipe. + + Documentation: https://docs.snowflake.com/en/sql-reference/info-schema/pipes + + The view's ``DEFINITION`` is the COPY statement the pipe runs, not + DDL; ``PATTERN`` is the copy option from that statement. Current + pipe status is deliberately not a column here: the page points at + ``SYSTEM$PIPE_STATUS`` for it, and a pipe that has never run would + otherwise read as a pipe that is stopped. + """ + return self._catalog_query( + [ + self._catalog_column("PIPE_CATALOG"), + self._catalog_column("PIPE_SCHEMA"), + self._catalog_column("PIPE_NAME"), + self._catalog_column("PIPE_OWNER"), + self._catalog_column("DEFINITION"), + self._catalog_column("IS_AUTOINGEST_ENABLED"), + self._catalog_column("NOTIFICATION_CHANNEL_NAME"), + self._catalog_column("CREATED"), + self._catalog_column("LAST_ALTERED"), + self._catalog_column("COMMENT"), + self._catalog_column("PATTERN"), + ], + self._information_schema_view("PIPES"), + [ + ComparisonPredicate( + self, + "eq", + self._catalog_column("PIPE_SCHEMA"), + Literal(self, expr.schema or ""), + ) + ], + [self._catalog_column("PIPE_NAME")], + ) + + def format_task_list_query_statement( + self, expr: SnowflakeTaskListQueryStatement + ) -> Tuple[str, tuple]: + """``SHOW TASKS``, scoped to the configured schema when there is one. + + Documentation: https://docs.snowflake.com/en/sql-reference/sql/show-tasks + + The Information Schema view index lists no ``TASKS`` view, so this is + the documented route. The ``SHOW`` statement renders the scope through + :class:`~...expression.show.SnowflakeShowStatement`; with no + configured schema it emits the bare command, whose documented default + is the current database. With a schema it emits ``IN SCHEMA`` so the + listing matches the schema the other families read. + + The rows are documented lowercase and the `SHOW` output is + privilege-filtered, both of which are recorded where the parser reads + the rows. + """ + from ..expression.show import SnowflakeShowObjectType, SnowflakeShowScope + + return SnowflakeShowStatement( + self, + SnowflakeShowObjectType.TASKS, + in_scope=SnowflakeShowScope.SCHEMA if expr.schema else None, + in_name=expr.schema or None, + ).to_sql() + + def format_stream_list_query_statement( + self, expr: SnowflakeStreamListQueryStatement + ) -> Tuple[str, tuple]: + """``SHOW STREAMS``, scoped to the configured schema when there is one. + + Documentation: https://docs.snowflake.com/en/sql-reference/sql/show-streams + + Same route and scope rule as :meth:`format_task_list_query_statement` + and for the same reason: no Information Schema view exists. The rows + are lowercase, and ``STALE`` / ``STALE_AFTER`` are the columns a caller + watches to know whether the stream can still be read. + """ + from ..expression.show import SnowflakeShowObjectType, SnowflakeShowScope + + return SnowflakeShowStatement( + self, + SnowflakeShowObjectType.STREAMS, + in_scope=SnowflakeShowScope.SCHEMA if expr.schema else None, + in_name=expr.schema or None, + ).to_sql() diff --git a/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_catalog_families_cli.py b/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_catalog_families_cli.py new file mode 100644 index 0000000..0340ad0 --- /dev/null +++ b/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_catalog_families_cli.py @@ -0,0 +1,440 @@ +# tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_catalog_families_cli.py +"""CLI wiring for ``routines`` / ``sequences`` / the catalog object types. + +No server is needed here. The backend is stubbed and the introspector records +each call, so a branch that reaches the wrong method, or drops the schema +argument, fails here rather than showing up as a plausible-looking listing. + +The advertised type list derives from ``_TYPE_CAPABILITIES``; every new type is +also checked against the dialect switch it names, so a family cannot be offered +without an answer behind it. +""" + +import argparse +import json +from typing import List, Optional + +import pytest + +from rhosocial.activerecord.backend.impl.snowflake.cli import introspect as introspect_cli +from rhosocial.activerecord.backend.impl.snowflake.dialect import SnowflakeDialect +from rhosocial.activerecord.backend.impl.snowflake.introspection import ( + SnowflakeCatalogObjectInfo, + SnowflakeRoutineInfo, +) +from rhosocial.activerecord.backend.introspection.types import SequenceInfo + +#: The new CLI type -> the introspector method it reaches. +NEW_TYPES = { + "routines": "list_routines", + "sequences": "list_sequences", + "stages": "list_stages", + "file-formats": "list_file_formats", + "pipes": "list_pipes", + "tasks": "list_tasks", + "streams": "list_streams", +} + +#: The switch each type is gated on. +NEW_SWITCHES = { + "routines": "supports_routine_introspection", + "sequences": "supports_sequence_introspection", + "stages": "supports_stage_introspection", + "file-formats": "supports_file_format_introspection", + "pipes": "supports_pipe_introspection", + "tasks": "supports_task_introspection", + "streams": "supports_stream_introspection", +} + +CONNECTION = ["--account", "acct"] + + +class _RecordingIntrospector: + def __init__(self, results: Optional[dict] = None) -> None: + self.results = results or {} + self.calls = [] + + def __getattr__(self, name: str): + def call(*args, **kwargs): + self.calls.append((name, args, kwargs)) + return self.results.get(name) + + return call + + +class _AsyncRecordingIntrospector(_RecordingIntrospector): + def __getattr__(self, name: str): + async def call(*args, **kwargs): + self.calls.append((name, args, kwargs)) + return self.results.get(name) + + return call + + +class _AllTrueDialect: + def __getattr__(self, name: str): + if name.startswith("supports_"): + return lambda: True + raise AttributeError(name) + + +class _RecordingProvider: + def __init__(self) -> None: + self.results: List[object] = [] + + def display_results(self, data, title=None): + self.results.append((title, data)) + + def display_connection_error(self, error): + pass + + def display_query_error(self, error): + pass + + +def build_parser(): + top = argparse.ArgumentParser() + subparsers = top.add_subparsers(dest="command") + introspect_cli.create_parser(subparsers) + return top + + +def _patch_backend(monkeypatch, introspector, dialect=None, use_async=False): + target = ( + introspect_cli.AsyncSnowflakeBackend.__name__ + if use_async + else introspect_cli.SnowflakeBackend.__name__ + ) + chosen = _AllTrueDialect if dialect is None else dialect + + class _Backend: + def __init__(self, connection_config=None): + self.connection_config = connection_config + self.introspector = introspector + self.dialect = chosen() + self._connection = object() + self.disconnected = False + + if use_async: + async def connect(self): + pass + + async def disconnect(self): + self.disconnected = True + else: + def connect(self): + pass + + def disconnect(self): + self.disconnected = True + + _Backend.connect = connect + _Backend.disconnect = disconnect + + monkeypatch.setattr(introspect_cli, target, _Backend) + monkeypatch.setattr( + introspect_cli, "resolve_connection_config_from_args", lambda _args: None + ) + return _Backend + + +def _run_sync(monkeypatch, introspector, argv, dialect=None): + monkeypatch.setattr( + introspect_cli, + "create_provider", + lambda fmt, ascii_borders=False: _RecordingProvider(), + ) + _patch_backend(monkeypatch, introspector, dialect=dialect) + introspect_cli.handle(build_parser().parse_args(argv)) + + +def _run_async(monkeypatch, introspector, argv, dialect=None): + monkeypatch.setattr( + introspect_cli, + "create_provider", + lambda fmt, ascii_borders=False: _RecordingProvider(), + ) + _patch_backend(monkeypatch, introspector, dialect=dialect, use_async=True) + introspect_cli.handle(build_parser().parse_args(argv + ["--use-async"])) + + +def _run_with_real_provider(monkeypatch, introspector, argv): + _patch_backend(monkeypatch, introspector) + introspect_cli.handle(build_parser().parse_args(argv)) + + +def _sample_result(method: str): + if method == "list_routines": + return [ + SnowflakeRoutineInfo( + name="ADD_ONE", + kind="FUNCTION", + schema="PUBLIC", + language="SQL", + argument_signature="(NUMBER)", + return_type="NUMBER", + owner="SYSADMIN", + comment="adds one", + ) + ] + if method == "list_sequences": + return [ + SequenceInfo( + name="ORDERS_ID_SEQ", + schema="PUBLIC", + data_type="NUMBER", + start_value=1, + increment=1, + extra={"snowflake_owner": "SYSADMIN", "snowflake_next_value": 4}, + ) + ] + return [ + SnowflakeCatalogObjectInfo( + name="OBJECT_1", + kind="STAGE", + catalog="MY_DB", + schema="PUBLIC", + owner="SYSADMIN", + comment="sample", + extra={"stage_url": "s3://bucket/path/"}, + ) + ] + + +# ================================================================== # +# Argument parsing and NAME rules +# ================================================================== # + + +class TestCatalogFamilyArguments: + @pytest.mark.parametrize("type_", list(NEW_TYPES)) + def test_every_new_type_is_advertised_and_parses(self, type_): + assert type_ in introspect_cli.INTROSPECT_TYPES + args = build_parser().parse_args(["introspect", type_] + CONNECTION) + assert args.type == type_ + assert args.name is None + assert args.use_async is False + + @pytest.mark.parametrize("type_", list(NEW_TYPES)) + def test_a_name_is_refused(self, type_, capsys): + with pytest.raises(SystemExit) as excinfo: + introspect_cli._refuse_name( + build_parser().parse_args(["introspect", type_, "orders"] + CONNECTION) + ) + assert excinfo.value.code == 2 + assert "takes no name" in capsys.readouterr().err + + @pytest.mark.parametrize("type_", list(NEW_TYPES)) + def test_the_refusal_happens_before_anything_connects(self, type_, monkeypatch): + introspector = _RecordingIntrospector() + with pytest.raises(SystemExit): + _run_sync( + monkeypatch, + introspector, + ["introspect", type_, "orders"] + CONNECTION, + ) + assert introspector.calls == [] + + def test_the_new_types_are_in_the_no_name_rule(self): + assert set(NEW_TYPES) <= set(introspect_cli._NO_NAME_TYPES) + + def test_the_help_lists_every_new_type(self, capsys): + with pytest.raises(SystemExit): + build_parser().parse_args(["introspect", "--help"]) + out = capsys.readouterr().out + for type_ in NEW_TYPES: + assert type_ in out + + @pytest.mark.parametrize("type_", list(NEW_TYPES)) + def test_the_advertised_list_is_the_capability_mapping(self, type_): + assert list(introspect_cli._TYPE_CAPABILITIES) == introspect_cli.INTROSPECT_TYPES + + +# ================================================================== # +# Capability gate +# ================================================================== # + + +class TestCatalogFamilyCapabilities: + @pytest.mark.parametrize("type_, switch", list(NEW_SWITCHES.items())) + def test_each_type_names_a_switch_the_dialect_declares(self, type_, switch): + assert introspect_cli._TYPE_CAPABILITIES[type_] == switch + assert hasattr(SnowflakeDialect, switch) + assert getattr(SnowflakeDialect(), switch)() is True + + @pytest.mark.parametrize("type_, switch", list(NEW_SWITCHES.items())) + def test_a_switch_that_answers_false_refuses(self, type_, switch, capsys): + class _Dialect: + pass + + setattr(_Dialect, switch, staticmethod(lambda: False)) + + class _Backend: + dialect = _Dialect() + + with pytest.raises(SystemExit) as excinfo: + introspect_cli._require_capability( + _Backend(), argparse.Namespace(type=type_) + ) + + assert excinfo.value.code == 2 + assert "not supported by this server" in capsys.readouterr().err + + @pytest.mark.parametrize("type_", list(NEW_TYPES)) + def test_the_real_dialect_passes_the_gate(self, type_, monkeypatch): + introspector = _RecordingIntrospector( + {NEW_TYPES[type_]: _sample_result(NEW_TYPES[type_])} + ) + # No SystemExit: the switch is True on the live dialect. + _run_sync( + monkeypatch, + introspector, + ["introspect", type_] + CONNECTION, + dialect=SnowflakeDialect, + ) + assert len(introspector.calls) == 1 + + +# ================================================================== # +# Synchronous dispatch +# ================================================================== # + + +class TestSyncCatalogFamilyDispatch: + @pytest.mark.parametrize( + "type_, method, title", + [ + ("routines", "list_routines", "Routines"), + ("sequences", "list_sequences", "Sequences"), + ("stages", "list_stages", "Stages"), + ("file-formats", "list_file_formats", "File Formats"), + ("pipes", "list_pipes", "Pipes"), + ("tasks", "list_tasks", "Tasks"), + ("streams", "list_streams", "Streams"), + ], + ) + def test_the_family_reaches_its_method_with_the_schema_slot( + self, type_, method, title, monkeypatch + ): + introspector = _RecordingIntrospector({method: _sample_result(method)}) + + _run_sync(monkeypatch, introspector, ["introspect", type_] + CONNECTION) + + assert introspector.calls == [(method, (), {"schema": None})] + + @pytest.mark.parametrize("type_", list(NEW_TYPES)) + def test_the_result_reaches_the_provider(self, type_, monkeypatch): + method = NEW_TYPES[type_] + introspector = _RecordingIntrospector({method: _sample_result(method)}) + provider = _RecordingProvider() + + monkeypatch.setattr( + introspect_cli, + "create_provider", + lambda fmt, ascii_borders=False: provider, + ) + _patch_backend(monkeypatch, introspector) + introspect_cli.handle( + build_parser().parse_args(["introspect", type_] + CONNECTION) + ) + + assert len(provider.results) == 1 + title, data = provider.results[0] + assert data + assert title is not None + + def test_routines_output_is_json_serialisable(self, monkeypatch, capsys): + introspector = _RecordingIntrospector( + {"list_routines": _sample_result("list_routines")} + ) + _run_with_real_provider( + monkeypatch, + introspector, + ["introspect", "routines", "-o", "json"] + CONNECTION, + ) + + payload = json.loads(capsys.readouterr().out) + assert payload[0]["name"] == "ADD_ONE" + assert payload[0]["kind"] == "FUNCTION" + assert payload[0]["language"] == "SQL" + assert payload[0]["argument_signature"] == "(NUMBER)" + assert payload[0]["return_type"] == "NUMBER" + + def test_sequences_output_keeps_core_fields_and_extras(self, monkeypatch, capsys): + introspector = _RecordingIntrospector( + {"list_sequences": _sample_result("list_sequences")} + ) + _run_with_real_provider( + monkeypatch, + introspector, + ["introspect", "sequences", "-o", "json"] + CONNECTION, + ) + + payload = json.loads(capsys.readouterr().out) + assert payload[0]["start_value"] == 1 + assert payload[0]["increment"] == 1 + assert payload[0]["extra"]["snowflake_next_value"] == 4 + + def test_catalog_output_keeps_the_stable_shape(self, monkeypatch, capsys): + introspector = _RecordingIntrospector( + {"list_stages": _sample_result("list_stages")} + ) + _run_with_real_provider( + monkeypatch, + introspector, + ["introspect", "stages", "-o", "json"] + CONNECTION, + ) + + payload = json.loads(capsys.readouterr().out) + assert payload[0]["kind"] == "STAGE" + assert payload[0]["schema"] == "PUBLIC" + assert payload[0]["extra"]["stage_url"] == "s3://bucket/path/" + + +# ================================================================== # +# Asynchronous dispatch +# ================================================================== # + + +class TestAsyncCatalogFamilyDispatch: + @pytest.mark.parametrize("type_, method", list(NEW_TYPES.items())) + def test_async_reaches_the_same_method_with_the_same_arguments( + self, type_, method, monkeypatch + ): + introspector = _AsyncRecordingIntrospector( + {method: _sample_result(method)} + ) + + _run_async(monkeypatch, introspector, ["introspect", type_] + CONNECTION) + + assert introspector.calls == [(method, (), {"schema": None})] + + async def test_async_result_reaches_the_provider(self): + introspector = _AsyncRecordingIntrospector( + {"list_tasks": [SnowflakeCatalogObjectInfo(name="T", kind="TASK")]} + ) + args = build_parser().parse_args( + ["introspect", "tasks", "--use-async"] + CONNECTION + ) + backend = _FakeAsyncBackend(introspector) + provider = _RecordingProvider() + + await introspect_cli._handle_introspect_async(args, backend, provider) + + provider_results = provider.results + assert provider_results[0][0] == "Tasks" + assert provider_results[0][1][0]["name"] == "T" + + +class _FakeAsyncBackend: + """The async handler's backend shape: awaited connect/disconnect.""" + + def __init__(self, introspector): + self.introspector = introspector + self._connection = None + + async def connect(self): + pass + + async def disconnect(self): + pass diff --git a/tests/rhosocial/activerecord_snowflake_test/feature/backend/introspection/test_routines_catalog_identity.py b/tests/rhosocial/activerecord_snowflake_test/feature/backend/introspection/test_routines_catalog_identity.py new file mode 100644 index 0000000..01759ed --- /dev/null +++ b/tests/rhosocial/activerecord_snowflake_test/feature/backend/introspection/test_routines_catalog_identity.py @@ -0,0 +1,917 @@ +# tests/rhosocial/activerecord_snowflake_test/feature/backend/introspection/test_routines_catalog_identity.py +"""Routines, sequences, catalog objects and the identity audit. + +Every test is offline. Rows are fabricated from the documented column names +and value domains; no Snowflake account is contacted, so a passing test pins +the statement and the parser, never the account's answer. + +Documentation the expectations were written from: + +* https://docs.snowflake.com/en/sql-reference/info-schema/functions +* https://docs.snowflake.com/en/sql-reference/info-schema/procedures +* https://docs.snowflake.com/en/sql-reference/info-schema/sequences +* https://docs.snowflake.com/en/sql-reference/info-schema/stages +* https://docs.snowflake.com/en/sql-reference/info-schema/file_formats +* https://docs.snowflake.com/en/sql-reference/info-schema/pipes +* https://docs.snowflake.com/en/sql-reference/sql/show-tasks +* https://docs.snowflake.com/en/sql-reference/sql/show-streams +* https://docs.snowflake.com/en/sql-reference/info-schema/columns +""" + +import inspect +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from rhosocial.activerecord.backend.impl.snowflake.dialect import SnowflakeDialect +from rhosocial.activerecord.backend.impl.snowflake.expression.introspection import ( + SnowflakeFileFormatListQueryStatement, + SnowflakeFunctionListQueryStatement, + SnowflakePipeListQueryStatement, + SnowflakeProcedureListQueryStatement, + SnowflakeSequenceListQueryStatement, + SnowflakeStageListQueryStatement, + SnowflakeStreamListQueryStatement, + SnowflakeTaskListQueryStatement, +) +from rhosocial.activerecord.backend.impl.snowflake.introspection import ( + AsyncSnowflakeIntrospector, + SnowflakeCatalogObjectInfo, + SnowflakeIntrospector, + SnowflakeIntrospectorMixin, + SnowflakeRoutineInfo, +) +from rhosocial.activerecord.backend.introspection.executor import IntrospectorExecutor +from rhosocial.activerecord.backend.introspection.types import SequenceInfo + +FUNCTIONS_SQL = ( + "SELECT FUNCTION_CATALOG, FUNCTION_SCHEMA, FUNCTION_NAME, FUNCTION_OWNER, " + "ARGUMENT_SIGNATURE, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, " + "CHARACTER_OCTET_LENGTH, NUMERIC_PRECISION, NUMERIC_PRECISION_RADIX, " + "NUMERIC_SCALE, FUNCTION_LANGUAGE, FUNCTION_DEFINITION, " + "VOLATILITY, IS_NULL_CALL, IS_SECURE, IS_MEMOIZABLE, IS_EXTERNAL, " + "IS_DATA_METRIC, IS_AGGREGATE, RUNTIME_VERSION, HANDLER, PACKAGES, " + "INSTALLED_PACKAGES, ARTIFACT_REPOSITORY, EXTERNAL_ACCESS_INTEGRATIONS, " + "SECRETS, CREATED, LAST_ALTERED, COMMENT, OWNER_ROLE_TYPE " + "FROM INFORMATION_SCHEMA.FUNCTIONS WHERE FUNCTION_SCHEMA = %s " + "ORDER BY FUNCTION_NAME, ARGUMENT_SIGNATURE" +) + +PROCEDURES_SQL = ( + "SELECT PROCEDURE_CATALOG, PROCEDURE_SCHEMA, PROCEDURE_NAME, PROCEDURE_OWNER, " + "ARGUMENT_SIGNATURE, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, " + "CHARACTER_OCTET_LENGTH, NUMERIC_PRECISION, NUMERIC_PRECISION_RADIX, " + "NUMERIC_SCALE, PROCEDURE_LANGUAGE, PROCEDURE_DEFINITION, " + "RUNTIME_VERSION, PACKAGES, INSTALLED_PACKAGES, ARTIFACT_REPOSITORY, " + "EXTERNAL_ACCESS_INTEGRATIONS, SECRETS, CREATED, LAST_ALTERED, COMMENT, " + "OWNER_ROLE_TYPE FROM INFORMATION_SCHEMA.PROCEDURES " + "WHERE PROCEDURE_SCHEMA = %s ORDER BY PROCEDURE_NAME, ARGUMENT_SIGNATURE" +) + +SEQUENCES_SQL = ( + "SELECT SEQUENCE_CATALOG, SEQUENCE_SCHEMA, SEQUENCE_NAME, SEQUENCE_OWNER, " + "DATA_TYPE, NUMERIC_PRECISION, NUMERIC_PRECISION_RADIX, NUMERIC_SCALE, " + "START_VALUE, MINIMUM_VALUE, MAXIMUM_VALUE, NEXT_VALUE, INCREMENT, " + "CYCLE_OPTION, CREATED, LAST_ALTERED, ORDERED, COMMENT " + "FROM INFORMATION_SCHEMA.SEQUENCES WHERE SEQUENCE_SCHEMA = %s " + "ORDER BY SEQUENCE_NAME" +) + +STAGES_SQL = ( + "SELECT STAGE_CATALOG, STAGE_SCHEMA, STAGE_NAME, STAGE_URL, STAGE_REGION, " + "STAGE_TYPE, STAGE_OWNER, COMMENT, CREATED, LAST_ALTERED " + "FROM INFORMATION_SCHEMA.STAGES WHERE STAGE_SCHEMA = %s " + "ORDER BY STAGE_NAME" +) + +FILE_FORMATS_SQL = ( + "SELECT FILE_FORMAT_CATALOG, FILE_FORMAT_SCHEMA, FILE_FORMAT_NAME, " + "FILE_FORMAT_OWNER, FILE_FORMAT_TYPE, RECORD_DELIMITER, FIELD_DELIMITER, " + "SKIP_HEADER, DATE_FORMAT, TIME_FORMAT, TIMESTAMP_FORMAT, BINARY_FORMAT, " + "ESCAPE, ESCAPE_UNENCLOSED_FIELD, TRIM_SPACE, " + "FIELD_OPTIONALLY_ENCLOSED_BY, NULL_IF, COMPRESSION, " + "ERROR_ON_COLUMN_COUNT_MISMATCH, CREATED, LAST_ALTERED, COMMENT " + "FROM INFORMATION_SCHEMA.FILE_FORMATS WHERE FILE_FORMAT_SCHEMA = %s " + "ORDER BY FILE_FORMAT_NAME" +) + +PIPES_SQL = ( + "SELECT PIPE_CATALOG, PIPE_SCHEMA, PIPE_NAME, PIPE_OWNER, DEFINITION, " + "IS_AUTOINGEST_ENABLED, NOTIFICATION_CHANNEL_NAME, CREATED, LAST_ALTERED, " + "COMMENT, PATTERN FROM INFORMATION_SCHEMA.PIPES " + "WHERE PIPE_SCHEMA = %s ORDER BY PIPE_NAME" +) + + +@pytest.fixture +def dialect(): + return SnowflakeDialect(version=(8, 32, 0)) + + +@pytest.fixture +def mock_backend(dialect): + backend = MagicMock() + backend._version = (8, 32, 0) + backend.config = MagicMock() + backend.config.schema = "PUBLIC" + backend.config.database = "MY_DB" + backend.dialect = dialect + return backend + + +@pytest.fixture +def sync_executor(): + return MagicMock(spec=IntrospectorExecutor) + + +@pytest.fixture +def sync_introspector(mock_backend, sync_executor): + return SnowflakeIntrospector(mock_backend, sync_executor) + + +@pytest.fixture +def async_executor(): + return AsyncMock() + + +@pytest.fixture +def async_introspector(mock_backend, async_executor): + return AsyncSnowflakeIntrospector(mock_backend, async_executor) + + +@pytest.fixture +def parser(): + return SnowflakeIntrospectorMixin.__new__(SnowflakeIntrospectorMixin) + + +def function_row(**overrides): + row = { + "FUNCTION_CATALOG": "MY_DB", + "FUNCTION_SCHEMA": "PUBLIC", + "FUNCTION_NAME": "ADD_ONE", + "FUNCTION_OWNER": "SYSADMIN", + "ARGUMENT_SIGNATURE": "(NUMBER)", + "DATA_TYPE": "NUMBER", + "FUNCTION_LANGUAGE": "SQL", + "FUNCTION_DEFINITION": "SELECT N + 1", + "VOLATILITY": "IMMUTABLE", + "IS_NULL_CALL": "NO", + "IS_SECURE": "NO", + "IS_MEMOIZABLE": "NO", + "IS_EXTERNAL": "NO", + "IS_DATA_METRIC": "NO", + "IS_AGGREGATE": "NO", + "RUNTIME_VERSION": None, + "HANDLER": None, + "PACKAGES": None, + "INSTALLED_PACKAGES": None, + "ARTIFACT_REPOSITORY": None, + "EXTERNAL_ACCESS_INTEGRATIONS": None, + "SECRETS": None, + "CREATED": datetime(2026, 1, 1), + "LAST_ALTERED": datetime(2026, 1, 2), + "COMMENT": "adds one", + "OWNER_ROLE_TYPE": "ROLE", + } + row.update(overrides) + return row + + +def procedure_row(**overrides): + row = { + "PROCEDURE_CATALOG": "MY_DB", + "PROCEDURE_SCHEMA": "PUBLIC", + "PROCEDURE_NAME": "LOAD_ORDERS", + "PROCEDURE_OWNER": "SYSADMIN", + "ARGUMENT_SIGNATURE": "(VARCHAR)", + "DATA_TYPE": "VARCHAR", + "PROCEDURE_LANGUAGE": "PYTHON", + "PROCEDURE_DEFINITION": "def main(session, source): ...", + "RUNTIME_VERSION": "3.11", + "PACKAGES": "snowflake-snowpark-python", + "INSTALLED_PACKAGES": "snowflake-snowpark-python,pandas", + "ARTIFACT_REPOSITORY": None, + "EXTERNAL_ACCESS_INTEGRATIONS": None, + "SECRETS": None, + "CREATED": datetime(2026, 1, 1), + "LAST_ALTERED": datetime(2026, 1, 2), + "COMMENT": "loads orders", + "OWNER_ROLE_TYPE": "ROLE", + } + row.update(overrides) + return row + + +def sequence_row(**overrides): + row = { + "SEQUENCE_CATALOG": "MY_DB", + "SEQUENCE_SCHEMA": "PUBLIC", + "SEQUENCE_NAME": "ORDERS_ID_SEQ", + "SEQUENCE_OWNER": "SYSADMIN", + "DATA_TYPE": "NUMBER", + "NUMERIC_PRECISION": 38, + "NUMERIC_PRECISION_RADIX": 10, + "NUMERIC_SCALE": 0, + "START_VALUE": "1", + "MINIMUM_VALUE": None, + "MAXIMUM_VALUE": None, + "NEXT_VALUE": "4", + "INCREMENT": "1", + "CYCLE_OPTION": None, + "CREATED": datetime(2026, 1, 1), + "LAST_ALTERED": datetime(2026, 1, 2), + "ORDERED": "YES", + "COMMENT": "order ids", + } + row.update(overrides) + return row + + +def stage_row(**overrides): + row = { + "STAGE_CATALOG": "MY_DB", + "STAGE_SCHEMA": "PUBLIC", + "STAGE_NAME": "ORDERS_STAGE", + "STAGE_URL": "s3://bucket/path/", + "STAGE_REGION": "us-east-1", + "STAGE_TYPE": "External Named", + "STAGE_OWNER": "SYSADMIN", + "COMMENT": "order files", + "CREATED": datetime(2026, 1, 1), + "LAST_ALTERED": datetime(2026, 1, 2), + } + row.update(overrides) + return row + + +def file_format_row(**overrides): + row = { + "FILE_FORMAT_CATALOG": "MY_DB", + "FILE_FORMAT_SCHEMA": "PUBLIC", + "FILE_FORMAT_NAME": "ORDERS_CSV", + "FILE_FORMAT_OWNER": "SYSADMIN", + "FILE_FORMAT_TYPE": "CSV", + "RECORD_DELIMITER": "\\n", + "FIELD_DELIMITER": ",", + "SKIP_HEADER": 1, + "DATE_FORMAT": "YYYY-MM-DD", + "TIME_FORMAT": None, + "TIMESTAMP_FORMAT": None, + "BINARY_FORMAT": None, + "ESCAPE": None, + "ESCAPE_UNENCLOSED_FIELD": None, + "TRIM_SPACE": "TRUE", + "FIELD_OPTIONALLY_ENCLOSED_BY": '"', + "NULL_IF": "NULL", + "COMPRESSION": "AUTO", + "ERROR_ON_COLUMN_COUNT_MISMATCH": "YES", + "CREATED": datetime(2026, 1, 1), + "LAST_ALTERED": datetime(2026, 1, 2), + "COMMENT": "orders csv", + } + row.update(overrides) + return row + + +def pipe_row(**overrides): + row = { + "PIPE_CATALOG": "MY_DB", + "PIPE_SCHEMA": "PUBLIC", + "PIPE_NAME": "ORDERS_PIPE", + "PIPE_OWNER": "SYSADMIN", + "DEFINITION": "COPY INTO ORDERS FROM @ORDERS_STAGE", + "IS_AUTOINGEST_ENABLED": "YES", + "NOTIFICATION_CHANNEL_NAME": "arn:aws:sqs:...", + "CREATED": datetime(2026, 1, 1), + "LAST_ALTERED": datetime(2026, 1, 2), + "COMMENT": "loads orders", + "PATTERN": ".*orders.*", + } + row.update(overrides) + return row + + +def task_row(**overrides): + """A ``SHOW TASKS`` row, in the lowercase shape the page documents.""" + row = { + "created_on": datetime(2026, 1, 1), + "name": "REFRESH_ORDERS", + "id": "1", + "database_name": "MY_DB", + "schema_name": "PUBLIC", + "owner": "SYSADMIN", + "comment": "refresh", + "warehouse": "COMPUTE_WH", + "schedule": "5 MINUTE", + "scheduling_mode": "FIXED", + "target_completion_interval": None, + "predecessors": "[]", + "state": "started", + "definition": "CALL LOAD_ORDERS()", + "condition": None, + "allow_overlapping_execution": "false", + "error_integration": None, + "success_integration": None, + "last_committed_on": None, + "last_suspended_on": None, + "last_suspended_reason": None, + "owner_role_type": "ROLE", + "config": None, + "task_relations": '{"Predecessors":[]}', + "execute_as_user": None, + "overlap_policy": "NO_OVERLAP", + } + row.update(overrides) + return row + + +def stream_row(**overrides): + """A ``SHOW STREAMS`` row, in the lowercase shape the page documents.""" + row = { + "created_on": datetime(2026, 1, 1), + "name": "ORDERS_STREAM", + "database_name": "MY_DB", + "schema_name": "PUBLIC", + "owner": "SYSADMIN", + "comment": "orders changes", + "table_name": "ORDERS", + "source_type": "Table", + "base_tables": None, + "type": "DELTA", + "stale": "false", + "mode": "DEFAULT", + "stale_after": datetime(2026, 2, 1), + "invalid_reason": "N/A", + "owner_role_type": "ROLE", + } + row.update(overrides) + return row + + +# ================================================================== # +# SQL text -- the rendered tree, pinned +# ================================================================== # + + +class TestCatalogFamilySQL: + @pytest.mark.parametrize( + "statement, expected_sql", + [ + (SnowflakeFunctionListQueryStatement, FUNCTIONS_SQL), + (SnowflakeProcedureListQueryStatement, PROCEDURES_SQL), + (SnowflakeSequenceListQueryStatement, SEQUENCES_SQL), + (SnowflakeStageListQueryStatement, STAGES_SQL), + (SnowflakeFileFormatListQueryStatement, FILE_FORMATS_SQL), + (SnowflakePipeListQueryStatement, PIPES_SQL), + ], + ) + def test_the_listing_is_exact(self, dialect, statement, expected_sql): + assert statement(dialect, schema="PUBLIC").to_sql() == ( + expected_sql, + ("PUBLIC",), + ) + + def test_the_schema_is_a_bind_value_not_an_identifier(self, dialect): + sql, params = SnowflakeFunctionListQueryStatement( + dialect, schema="PUBLIC'); DROP TABLE users;--" + ).to_sql() + assert params == ("PUBLIC'); DROP TABLE users;--",) + assert "DROP TABLE" not in sql + + def test_tasks_and_streams_use_show_with_a_schema_scope(self, dialect): + assert SnowflakeTaskListQueryStatement(dialect, schema="PUBLIC").to_sql() == ( + 'SHOW TASKS IN SCHEMA "PUBLIC"', + (), + ) + assert SnowflakeStreamListQueryStatement( + dialect, schema="PUBLIC" + ).to_sql() == ('SHOW STREAMS IN SCHEMA "PUBLIC"', ()) + + def test_tasks_and_streams_keep_the_bare_show_without_a_schema(self, dialect): + assert SnowflakeTaskListQueryStatement(dialect).to_sql() == ( + "SHOW TASKS", + (), + ) + assert SnowflakeStreamListQueryStatement(dialect).to_sql() == ( + "SHOW STREAMS", + (), + ) + + @pytest.mark.parametrize( + "statement", + [ + SnowflakeFunctionListQueryStatement, + SnowflakeProcedureListQueryStatement, + SnowflakeSequenceListQueryStatement, + SnowflakeStageListQueryStatement, + SnowflakeFileFormatListQueryStatement, + SnowflakePipeListQueryStatement, + SnowflakeTaskListQueryStatement, + SnowflakeStreamListQueryStatement, + ], + ) + def test_every_statement_names_a_dialect_formatter(self, dialect, statement): + method = statement(dialect, schema="PUBLIC").format_method + assert method.endswith("_query_statement") + assert callable(getattr(dialect, method)), statement.__name__ + + +# ================================================================== # +# Routines and sequences +# ================================================================== # + + +class TestRoutineParsing: + def test_a_function_row_maps_to_the_routine_shape(self, parser): + [routine] = parser.parse_routines([function_row()], "FUNCTION", "PUBLIC") + assert routine == SnowflakeRoutineInfo( + name="ADD_ONE", + kind="FUNCTION", + schema="PUBLIC", + catalog="MY_DB", + language="SQL", + argument_signature="(NUMBER)", + return_type="NUMBER", + owner="SYSADMIN", + comment="adds one", + created=datetime(2026, 1, 1), + last_altered=datetime(2026, 1, 2), + definition="SELECT N + 1", + extra={ + "character_maximum_length": None, + "character_octet_length": None, + "numeric_precision": None, + "numeric_precision_radix": None, + "numeric_scale": None, + "volatility": "IMMUTABLE", + "is_null_call": False, + "is_secure": False, + "is_memoizable": False, + "is_external": False, + "is_data_metric": False, + "is_aggregate": False, + "runtime_version": None, + "handler": None, + "packages": None, + "installed_packages": None, + "artifact_repository": None, + "external_access_integrations": None, + "secrets": None, + "owner_role_type": "ROLE", + }, + ) + + def test_a_procedure_row_maps_to_the_same_shape(self, parser): + [routine] = parser.parse_routines([procedure_row()], "PROCEDURE", "PUBLIC") + assert routine.name == "LOAD_ORDERS" + assert routine.kind == "PROCEDURE" + assert routine.language == "PYTHON" + assert routine.argument_signature == "(VARCHAR)" + assert routine.return_type == "VARCHAR" + assert routine.definition.startswith("def main") + assert routine.extra["runtime_version"] == "3.11" + assert routine.extra["packages"] == "snowflake-snowpark-python" + + def test_function_only_columns_are_absent_on_a_procedure(self, parser): + """The stable shape keeps the key; the value is ``None``, not a KeyError.""" + [routine] = parser.parse_routines([procedure_row()], "PROCEDURE", "PUBLIC") + assert routine.extra["volatility"] is None + assert routine.extra["is_aggregate"] is None + + def test_routine_keys_are_case_insensitive(self, parser): + row = {key.lower(): value for key, value in function_row().items()} + [routine] = parser.parse_routines([row], "FUNCTION", "PUBLIC") + assert routine.name == "ADD_ONE" + + def test_an_unknown_routine_kind_reads_the_prefix_it_is_given(self, parser): + [routine] = parser.parse_routines( + [{"FUNCTION_NAME": "F", "FUNCTION_SCHEMA": "PUBLIC"}], "FUNCTION", "PUBLIC" + ) + assert routine.kind == "FUNCTION" + assert routine.name == "F" + + +class TestSequenceParsing: + def test_a_sequence_row_uses_core_sequence_info_plus_extras(self, parser): + [sequence] = parser.parse_sequences([sequence_row()], "PUBLIC") + assert sequence == SequenceInfo( + name="ORDERS_ID_SEQ", + schema="PUBLIC", + data_type="NUMBER", + start_value=1, + minimum_value=None, + maximum_value=None, + increment=1, + cycle=None, + cache_size=None, + last_value=None, + extra={ + "snowflake_catalog": "MY_DB", + "snowflake_owner": "SYSADMIN", + "snowflake_next_value": 4, + "snowflake_ordered": True, + "snowflake_numeric_precision": 38, + "snowflake_numeric_precision_radix": 10, + "snowflake_numeric_scale": 0, + "snowflake_created": datetime(2026, 1, 1), + "snowflake_last_altered": datetime(2026, 1, 2), + "snowflake_comment": "order ids", + }, + ) + + def test_next_value_is_not_mislabelled_last_value(self, parser): + [sequence] = parser.parse_sequences([sequence_row()], "PUBLIC") + assert sequence.last_value is None + assert sequence.extra["snowflake_next_value"] == 4 + + def test_the_documented_not_applicable_fields_stay_none(self, parser): + [sequence] = parser.parse_sequences( + [ + sequence_row( + MINIMUM_VALUE="not-a-number", + MAXIMUM_VALUE="not-a-number", + CYCLE_OPTION="not-applicable", + ) + ], + "PUBLIC", + ) + assert sequence.minimum_value is None + assert sequence.maximum_value is None + assert sequence.cycle is None + + def test_a_hypothetical_cycle_flag_still_reads(self, parser): + [sequence] = parser.parse_sequences( + [sequence_row(CYCLE_OPTION="NO")], "PUBLIC" + ) + assert sequence.cycle is False + + +# ================================================================== # +# The catalog objects: stages, file formats, pipes, tasks, streams +# ================================================================== # + + +class TestCatalogObjectParsing: + def test_stage_row_maps_to_the_catalog_shape(self, parser): + [stage] = parser.parse_stages([stage_row()], "PUBLIC") + assert stage == SnowflakeCatalogObjectInfo( + name="ORDERS_STAGE", + kind="STAGE", + catalog="MY_DB", + schema="PUBLIC", + owner="SYSADMIN", + comment="order files", + created=datetime(2026, 1, 1), + last_altered=datetime(2026, 1, 2), + extra={ + "stage_url": "s3://bucket/path/", + "stage_region": "us-east-1", + "stage_type": "External Named", + }, + ) + + def test_file_format_row_keeps_its_options(self, parser): + [file_format] = parser.parse_file_formats([file_format_row()], "PUBLIC") + assert file_format.kind == "FILE FORMAT" + assert file_format.extra["file_format_type"] == "CSV" + assert file_format.extra["field_delimiter"] == "," + assert file_format.extra["skip_header"] == 1 + assert file_format.extra["trim_space"] is True + assert file_format.extra["error_on_column_count_mismatch"] is True + + def test_pipe_row_keeps_the_copy_definition_and_pattern(self, parser): + [pipe] = parser.parse_pipes([pipe_row()], "PUBLIC") + assert pipe.kind == "PIPE" + assert pipe.extra["definition"].startswith("COPY INTO") + assert pipe.extra["is_autoingest_enabled"] is True + assert pipe.extra["pattern"] == ".*orders.*" + + def test_task_row_maps_lowercase_keys(self, parser): + [task] = parser.parse_tasks([task_row()]) + assert task == SnowflakeCatalogObjectInfo( + name="REFRESH_ORDERS", + kind="TASK", + catalog="MY_DB", + schema="PUBLIC", + owner="SYSADMIN", + comment="refresh", + created=datetime(2026, 1, 1), + last_altered=None, + extra={ + "id": "1", + "warehouse": "COMPUTE_WH", + "schedule": "5 MINUTE", + "scheduling_mode": "FIXED", + "target_completion_interval": None, + "predecessors": "[]", + "state": "started", + "definition": "CALL LOAD_ORDERS()", + "condition": None, + "allow_overlapping_execution": False, + "error_integration": None, + "success_integration": None, + "last_committed_on": None, + "last_suspended_on": None, + "last_suspended_reason": None, + "task_relations": '{"Predecessors":[]}', + "execute_as_user": None, + "overlap_policy": "NO_OVERLAP", + "owner_role_type": "ROLE", + "config": None, + }, + ) + + def test_stream_row_reports_staleness(self, parser): + [stream] = parser.parse_streams([stream_row()]) + assert stream.kind == "STREAM" + assert stream.extra["table_name"] == "ORDERS" + assert stream.extra["stale"] is False + assert stream.extra["stale_after"] == datetime(2026, 2, 1) + assert stream.extra["mode"] == "DEFAULT" + + def test_stale_true_reads_as_true(self, parser): + [stream] = parser.parse_streams([stream_row(stale="TRUE")]) + assert stream.extra["stale"] is True + + def test_show_values_are_not_forced_into_datetimes(self, parser): + """The driver may hand back text; the parser must not crash on it.""" + [task] = parser.parse_tasks([task_row(created_on="2026-01-01T00:00:00Z")]) + assert task.created == "2026-01-01T00:00:00Z" + + +# ================================================================== # +# Identity audit +# ================================================================== # + + +def column_row(**overrides): + row = { + "COLUMN_NAME": "id", + "ORDINAL_POSITION": 1, + "COLUMN_DEFAULT": None, + "IS_NULLABLE": "NO", + "DATA_TYPE": "NUMBER", + "NUMERIC_PRECISION": 38, + "NUMERIC_SCALE": 0, + "IS_IDENTITY": "NO", + "IDENTITY_GENERATION": None, + "IDENTITY_START": None, + "IDENTITY_INCREMENT": None, + "IDENTITY_ORDERED": None, + "KIND": "COLUMN", + "EXPRESSION": None, + } + row.update(overrides) + return row + + +class TestIdentityAudit: + @pytest.fixture + def mixin(self, mock_backend): + mixin = SnowflakeIntrospectorMixin() + mixin._backend = mock_backend + return mixin + + def test_an_identity_column_reports_is_auto_increment(self, mixin): + [column] = mixin.parse_columns( + [ + column_row( + IS_IDENTITY="YES", + IDENTITY_GENERATION="BY DEFAULT", + IDENTITY_START="1", + IDENTITY_INCREMENT="1", + IDENTITY_ORDERED="YES", + ) + ], + "orders", + "PUBLIC", + ) + assert column.is_auto_increment is True + assert column.extra["snowflake_auto_increment_source"] == "identity" + assert column.extra["snowflake_identity"] is True + assert column.extra["snowflake_identity_generation"] == "BY DEFAULT" + assert column.extra["snowflake_identity_start"] == 1 + assert column.extra["snowflake_identity_increment"] == 1 + assert column.extra["snowflake_identity_ordered"] is True + + def test_a_plain_column_is_not_auto_increment(self, mixin): + [column] = mixin.parse_columns([column_row()], "orders", "PUBLIC") + assert column.is_auto_increment is False + assert column.extra["snowflake_auto_increment_source"] is None + + def test_a_sequence_backed_default_is_reported(self, mixin): + [column] = mixin.parse_columns( + [column_row(COLUMN_DEFAULT="ORDERS_ID_SEQ.NEXTVAL")], + "orders", + "PUBLIC", + ) + assert column.is_auto_increment is True + assert column.extra["snowflake_auto_increment_source"] == "sequence_default" + assert column.extra["snowflake_sequence_backed"] is True + + def test_identity_wins_over_the_default_when_both_are_present(self, mixin): + [column] = mixin.parse_columns( + [ + column_row( + IS_IDENTITY="YES", + COLUMN_DEFAULT="ORDERS_ID_SEQ.NEXTVAL", + ) + ], + "orders", + "PUBLIC", + ) + assert column.extra["snowflake_auto_increment_source"] == "identity" + + def test_a_virtual_column_reports_its_expression(self, mixin): + [column] = mixin.parse_columns( + [column_row(KIND="VIRTUAL_COLUMN", EXPRESSION="AMOUNT * 2")], + "orders", + "PUBLIC", + ) + assert column.is_generated is True + assert column.generated_expression == "AMOUNT * 2" + + def test_a_regular_column_has_no_generated_expression(self, mixin): + [column] = mixin.parse_columns( + [column_row(KIND="COLUMN", EXPRESSION="IGNORED")], + "orders", + "PUBLIC", + ) + assert column.is_generated is False + assert column.generated_expression is None + + def test_identity_spelling_is_case_insensitive(self, mixin): + [column] = mixin.parse_columns( + [column_row(IS_IDENTITY=" yes ")], "orders", "PUBLIC" + ) + assert column.is_auto_increment is True + + def test_a_default_without_nextval_is_not_sequence_backed(self, mixin): + [column] = mixin.parse_columns( + [column_row(COLUMN_DEFAULT="0")], "orders", "PUBLIC" + ) + assert column.is_auto_increment is False + assert column.extra["snowflake_sequence_backed"] is False + + +# ================================================================== # +# Sync public API +# ================================================================== # + + +class TestSyncCatalogIntrospection: + def test_list_routines_reads_both_views_and_sorts( + self, sync_introspector, sync_executor + ): + sync_executor.execute.side_effect = [ + [function_row(FUNCTION_NAME="Z_FN")], + [procedure_row(PROCEDURE_NAME="A_PROC")], + ] + routines = sync_introspector.list_routines() + + assert [routine.name for routine in routines] == ["Z_FN", "A_PROC"] + assert [routine.kind for routine in routines] == ["FUNCTION", "PROCEDURE"] + sync_executor.execute.assert_any_call(FUNCTIONS_SQL, ("PUBLIC",)) + sync_executor.execute.assert_any_call(PROCEDURES_SQL, ("PUBLIC",)) + + def test_list_routines_second_call_is_a_cache_hit( + self, sync_introspector, sync_executor + ): + sync_executor.execute.side_effect = [[function_row()], [procedure_row()]] + first = sync_introspector.list_routines() + second = sync_introspector.list_routines() + assert first is second + assert sync_executor.execute.call_count == 2 + + def test_list_sequences_uses_the_configured_schema_and_caches( + self, sync_introspector, sync_executor + ): + sync_executor.execute.return_value = [sequence_row()] + first = sync_introspector.list_sequences() + second = sync_introspector.list_sequences() + assert first is second + assert first[0].name == "ORDERS_ID_SEQ" + sync_executor.execute.assert_called_once_with(SEQUENCES_SQL, ("PUBLIC",)) + + @pytest.mark.parametrize( + "method, statement, expected_sql", + [ + ("list_stages", SnowflakeStageListQueryStatement, STAGES_SQL), + ( + "list_file_formats", + SnowflakeFileFormatListQueryStatement, + FILE_FORMATS_SQL, + ), + ("list_pipes", SnowflakePipeListQueryStatement, PIPES_SQL), + ], + ) + def test_catalog_view_families_execute_their_statement( + self, sync_introspector, sync_executor, method, statement, expected_sql + ): + sync_executor.execute.return_value = [] + getattr(sync_introspector, method)() + sync_executor.execute.assert_called_once_with(expected_sql, ("PUBLIC",)) + + def test_tasks_and_streams_execute_scoped_show( + self, sync_introspector, sync_executor + ): + sync_executor.execute.return_value = [] + sync_introspector.list_tasks() + sync_executor.execute.assert_called_with( + 'SHOW TASKS IN SCHEMA "PUBLIC"', () + ) + sync_introspector.list_streams() + sync_executor.execute.assert_called_with( + 'SHOW STREAMS IN SCHEMA "PUBLIC"', () + ) + + def test_catalog_families_do_not_share_cache_entries( + self, sync_introspector, sync_executor + ): + """The family name is part of the key, so five scope-sharing families + cannot answer each other's questions.""" + sync_executor.execute.return_value = [stage_row()] + sync_introspector.list_stages() + sync_introspector.list_pipes() + sync_introspector.list_tasks() + assert sync_executor.execute.call_count == 3 + + +# ================================================================== # +# Async public API +# ================================================================== # + + +class TestAsyncCatalogIntrospection: + async def test_async_methods_await_the_same_statements( + self, async_introspector, async_executor + ): + async_executor.execute.side_effect = [ + [function_row()], + [procedure_row()], + [sequence_row()], + [stage_row()], + [file_format_row()], + [pipe_row()], + [task_row()], + [stream_row()], + ] + + routines = await async_introspector.list_routines() + sequences = await async_introspector.list_sequences() + stages = await async_introspector.list_stages() + file_formats = await async_introspector.list_file_formats() + pipes = await async_introspector.list_pipes() + tasks = await async_introspector.list_tasks() + streams = await async_introspector.list_streams() + + assert len(routines) == 2 + assert sequences[0].name == "ORDERS_ID_SEQ" + assert stages[0].kind == "STAGE" + assert file_formats[0].kind == "FILE FORMAT" + assert pipes[0].kind == "PIPE" + assert tasks[0].kind == "TASK" + assert streams[0].kind == "STREAM" + assert async_executor.execute.await_count == 8 + + async def test_async_catalog_listing_second_call_is_a_cache_hit( + self, async_introspector, async_executor + ): + async_executor.execute.return_value = [stage_row()] + first = await async_introspector.list_stages() + second = await async_introspector.list_stages() + assert first is second + assert async_executor.execute.await_count == 1 + + +# ================================================================== # +# Value semantics and sync/async symmetry +# ================================================================== # + + +class TestCatalogValueSemantics: + def test_routine_and_catalog_values_are_frozen(self): + from dataclasses import FrozenInstanceError + + with pytest.raises(FrozenInstanceError): + SnowflakeRoutineInfo(name="F", kind="FUNCTION").name = "G" # type: ignore[misc] + with pytest.raises(FrozenInstanceError): + SnowflakeCatalogObjectInfo(name="S", kind="STAGE").kind = "PIPE" # type: ignore[misc] + + @pytest.mark.parametrize( + "method", + [ + "list_routines", + "list_sequences", + "list_stages", + "list_file_formats", + "list_pipes", + "list_tasks", + "list_streams", + ], + ) + def test_sync_and_async_methods_have_identical_signatures(self, method): + assert inspect.signature( + getattr(SnowflakeIntrospector, method) + ) == inspect.signature(getattr(AsyncSnowflakeIntrospector, method)) From b2ea3d9e6b855e72ff37648908cd54033b489289 Mon Sep 17 00:00:00 2001 From: Zhao vistart Date: Mon, 9 Nov 2026 16:33:41 +0800 Subject: [PATCH 2/4] ci(snowflake): test against testsuite release/v1.0.0.dev17 --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c6b8354..30b3b5a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: pip install -r requirements.txt pip install -r requirements-dev.txt pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" # NOSONAR - pip install git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/extract-epoch#egg=rhosocial-activerecord-testsuite + pip install git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17#egg=rhosocial-activerecord-testsuite - name: Install package run: | @@ -85,7 +85,7 @@ jobs: pip install -r requirements.txt pip install -r requirements-dev.txt pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" # NOSONAR - pip install git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/extract-epoch#egg=rhosocial-activerecord-testsuite + pip install git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17#egg=rhosocial-activerecord-testsuite - name: Install package run: | @@ -118,7 +118,7 @@ jobs: pip install -r requirements.txt pip install -r requirements-dev.txt pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" # NOSONAR - pip install git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/extract-epoch#egg=rhosocial-activerecord-testsuite + pip install git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17#egg=rhosocial-activerecord-testsuite - name: Install package run: | From da737a1c699255de677e3e24711ce2ee1e35c4c9 Mon Sep 17 00:00:00 2001 From: Zhao vistart Date: Mon, 9 Nov 2026 17:15:14 +0800 Subject: [PATCH 3/4] refactor(snowflake): stop advertising ddl introspection --- docs/introspection.md | 11 ++- .../backend/impl/snowflake/cli/README.md | 38 ++++----- .../backend/impl/snowflake/cli/introspect.py | 65 +++------------ .../backend/cli/test_cli_introspect.py | 8 ++ .../cli/test_introspect_families_cli.py | 80 +++---------------- 5 files changed, 56 insertions(+), 146 deletions(-) diff --git a/docs/introspection.md b/docs/introspection.md index d09e2ed..09d1424 100644 --- a/docs/introspection.md +++ b/docs/introspection.md @@ -66,7 +66,7 @@ The answers most easily mistaken for a gap in this backend, rather than a proper `supports_trigger_introspection()` is `False` because Snowflake has no trigger object to ask about. The dialect still has a trigger formatter -- it renders a statement that cannot match a row (`SELECT 1 WHERE 1 = 0`) -- so a caller that ignores the probe receives an empty listing from a real statement rather than an error. The probe is the answer, though: the question has no answer because the object does not exist. The CLI does not offer a `triggers` type at all. -The operational families -- `unused-indexes`, `table-stats`, `runtime` and `index-stats` -- read counters of a shape this engine does not publish to a session: per-index access counts and per-table I/O counters. `dependencies` needs a declared edge catalogue, and Snowflake's catalogue does not record which view, task or stage mentions which table. `ddl` needs the engine's own retained text for an object; this backend reads no such text and refuses to assemble one, because an assembled `CREATE` would read as authoritative while being this library's guess. `ddl_extraction_native` is `False` for the same reason. All six switches are constants on this dialect: unlike a feature that appears in a later release, these answers do not vary by server version. +The operational families -- `unused-indexes`, `table-stats`, `runtime` and `index-stats` -- read counters of a shape this engine does not publish to a session: per-index access counts and per-table I/O counters. `dependencies` needs a declared edge catalogue, and Snowflake's catalogue does not record which view, task or stage mentions which table. `ddl` needs the engine's own retained text for an object; this backend reads no such text and refuses to assemble one, because an assembled `CREATE` would read as authoritative while being this library's guess. `ddl_extraction_native` is `False` for the same reason. All six switches are constants on this dialect: unlike a feature that appears in a later release, these answers do not vary by server version. Because `supports_ddl_extraction()` is permanently `False`, the CLI does not offer a `ddl` type; the capability and the DDL formatter stay on the dialect for a future real implementation. `supports_partition_info()` stays `False` by inheritance. Snowflake's own partition questions live under their own switches, and `supports_partition_metadata_introspection()` answers `False` because the dialect has a formatter for the family but no introspector method reads it. The CLI excludes `partition-metadata` for that reason, rather than offering a type with no answer behind it. @@ -213,7 +213,7 @@ The four billed tiers are `ACTIVE_BYTES`, `TIME_TRAVEL_BYTES`, `FAILSAFE_BYTES` ## Operational Families -None of the six answers here. Each probe is `False`, each is a constant on the dialect, and the CLI refuses the type (exit 2) instead of printing an empty listing. +None of the six answers here. Each probe is `False` and each is a constant on the dialect. Five are CLI types, refused (exit 2) instead of printing an empty listing; `ddl` is not offered as a type at all, because its capability is permanently `False`. | Family | Probe | Why this backend declines | |--------|-------|---------------------------| @@ -351,7 +351,7 @@ The invocation requires `--account` and/or `--database`; connection parameters c ### Introspection Types -Twenty-six types. Seventeen are catalogue questions, six are families this backend declines, and three are the data families. +Twenty-five types. Seventeen are catalogue questions, five are families this backend declines, and three are the data families. | Type | Description | Name | |------|-------------|------| @@ -377,12 +377,11 @@ Twenty-six types. Seventeen are catalogue questions, six are families this backe | `runtime` | Declined: `False` | refused | | `index-stats` | Declined: `False` | refused | | `dependencies` | Declined: `False` | optional (object) | -| `ddl` | Declined: `False` | required (table) | | `column-profiles` | What each column of a table holds, `source='aggregate'` | required (table) | | `value-samples` | Most frequent values of one column | required (table) | | `join-coverage` | Overlap of two columns' value domains | required (left table) | -The six declined types are advertised rather than omitted, and refused at runtime by the switch behind them (exit 2) before the introspector is reached. The advertised list is derived from the same capability map the handler consults, so a family cannot be answerable and unaskable at once. +The five declined types are advertised rather than omitted, and refused at runtime by the switch behind them (exit 2) before the introspector is reached. `ddl` is not among them: its capability is permanently `False`, so the CLI does not advertise it. The advertised list is derived from the same capability map the handler consults, so a family cannot be answerable and unaskable at once. A `NAME` a type cannot use is refused (exit 2) rather than ignored, and a required one that is missing exits 1 -- a typo and a misfiled question are different failures. `value-samples` also requires `--column`; `join-coverage` requires `--column`, `--right-table` and `--right-column`, with the left table given as `NAME`. `--limit` is forwarded to `sample_column_values()` only when it is given; without it the introspector's own default answers, so the number is not written in two places. A flag the type does not read is refused (exit 2). @@ -396,7 +395,7 @@ python -m rhosocial.activerecord.backend.impl.snowflake introspect table-storage --account myaccount --database mydb # Declined families: exit 2, no query run -python -m rhosocial.activerecord.backend.impl.snowflake introspect ddl orders \ +python -m rhosocial.activerecord.backend.impl.snowflake introspect unused-indexes \ --account myaccount --database mydb # The data families diff --git a/src/rhosocial/activerecord/backend/impl/snowflake/cli/README.md b/src/rhosocial/activerecord/backend/impl/snowflake/cli/README.md index 4ea03e4..936e903 100644 --- a/src/rhosocial/activerecord/backend/impl/snowflake/cli/README.md +++ b/src/rhosocial/activerecord/backend/impl/snowflake/cli/README.md @@ -4,7 +4,7 @@ The `introspect` subcommand asks a live Snowflake account what it holds, and prints the answer. Every question runs through the backend's introspector, so what the CLI reports is what the library sees — the same objects, the same types, the same parsing — rather than a second implementation that can drift -from it. Twenty-six types are advertised, derived from `_TYPE_CAPABILITIES` in +from it. Twenty-five types are advertised, derived from `_TYPE_CAPABILITIES` in the source rather than written out a second time, so a family cannot be answerable and unaskable at once. @@ -14,10 +14,10 @@ python -m rhosocial.activerecord.backend.impl.snowflake introspect [NAME] ## What it can ask about -Twenty-six types, in the order `--help` prints them. The catalogue families -answer True for this dialect; the operational and one-object families are -offered and gated on their switch, so a run is refused (exit 2) rather than -answered with an empty list. +Twenty-five types, in the order `--help` prints them. The catalogue families +answer True for this dialect; the operational families are offered and gated on +their switch, so a run is refused (exit 2) rather than answered with an empty +list. | Type | `NAME` | Reads | Notes | | :--- | :--- | :--- | :--- | @@ -43,13 +43,12 @@ answered with an empty list. | `runtime` | — | refused here | Offered and gated on `supports_runtime_stats`, which answers False here. | | `index-stats` | — | refused here | Offered and gated on `supports_index_stats`, which answers False here. | | `dependencies` | object, optional | refused here | Offered and gated on `supports_object_dependencies` (False here); a name narrows the graph to that object's edges. | -| `ddl` | table, required | refused here | Offered and gated on `supports_ddl_extraction` (False here). | | `column-profiles` | table, required | the rows themselves | Answers True here: the profile is aggregated from the rows, so every figure is exact and paid for in time. | | `value-samples` | table, required | the rows themselves | Answers True here, read from the rows; `--column` names the column and `--limit` caps the count (default 100). | | `join-coverage` | left table, required | the rows themselves | Answers True here: how much two columns' value domains overlap. `--column`, `--right-table` and `--right-column` are required. | `NAME` is required for `columns`, `indexes`, `foreign-keys`, -`table-storage-metrics`, `ddl`, `column-profiles`, `value-samples` and +`table-storage-metrics`, `column-profiles`, `value-samples` and `join-coverage`. For `views` and `dependencies` it is optional: a name asks for that one view, or narrows the graph to that object's edges. The list types — `tables`, `database`, `databases`, `schemas`, `warehouses`, `routines`, @@ -66,12 +65,15 @@ unexpected) and also "the named view or table is not there"; **2** is a request this backend cannot serve — a type whose capability switch is off, or a `NAME` on a list type. -Two absences worth naming: +Three absences worth naming: * **No `triggers` type.** Snowflake has no triggers, so there is no object to ask about. That is an engine fact, not a gap in the CLI. * **No `partition-metadata` type.** The dialect carries a formatter for it, but no introspector method reads it, so there is no answer behind the type. +* **No `ddl` type.** DDL extraction is not offered by this backend: + `supports_ddl_extraction()` answers False, and the dialect refuses to + assemble text that would read as the engine's own. `table-storage-metrics` is a type of its own rather than a column of `tables` for two reasons. Its statement joins two views, so folding it into the listing @@ -81,16 +83,16 @@ for this role" — a `None` the command can state out loud, which a column on th listing could not. An exit of 1 on this type is therefore **not** "the table does not exist". -The nine families beyond the catalogue are offered for a different reason than -the ten above: they are questions the *protocol* asks every dialect, and -Snowflake's answer is a real answer, not a gap. Six of them — `unused-indexes`, -`table-stats`, `runtime`, `index-stats`, `dependencies` and `ddl` — are gated on -a switch that answers `False` here, so asking them exits 2 with "not supported by -this server"; the remaining three (`column-profiles`, `value-samples`, -`join-coverage`) read the rows themselves and answer True. Offering a gated type -at all is the point: the dialect's answer is "this engine cannot tell you", -which is a fact worth letting a caller ask for and be told, and is not the same -as an empty list that would read as "it has none". +The eight families beyond the catalogue are offered for a different reason than +the catalogue questions above: they are questions the *protocol* asks every +dialect, and Snowflake's answer is a real answer, not a gap. Five of them — +`unused-indexes`, `table-stats`, `runtime`, `index-stats` and `dependencies` — +are gated on a switch that answers `False` here, so asking them exits 2 with +"not supported by this server"; the remaining three (`column-profiles`, +`value-samples`, `join-coverage`) read the rows themselves and answer True. +Offering a gated type at all is the point: the dialect's answer is "this engine +cannot tell you", which is a fact worth letting a caller ask for and be told, and +is not the same as an empty list that would read as "it has none". ## Options diff --git a/src/rhosocial/activerecord/backend/impl/snowflake/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/snowflake/cli/introspect.py index 4730151..7a4d973 100644 --- a/src/rhosocial/activerecord/backend/impl/snowflake/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/snowflake/cli/introspect.py @@ -6,9 +6,10 @@ a second time -- so a family cannot be answerable and unaskable at once. The three families that read the rows themselves (data profiles, value samples, join coverage) are offered because the dialect answers them by aggregating; the -six that a catalogue or a statistics collector answers elsewhere are offered +five that a catalogue or a statistics collector answers elsewhere are offered too, and the switch that names them is what decides at runtime, because the -*server* in front of us may not have what the switch describes. +*server* in front of us may not have what the switch describes. DDL extraction +is not offered: its capability is permanently false here. A NAME or a flag the type has no question for is refused (exit 2) rather than dropped, and a required one that is missing exits 1. A probe that answers @@ -79,10 +80,9 @@ "index-stats": "supports_index_stats", "dependencies": "supports_object_dependencies", # ---- the one-object families ---- - # The DDL that recreates a named table, the profiles of one table and the - # samples of one of its columns. The dialect answers these by aggregating - # the rows themselves, so every figure is exact and paid for in time. - "ddl": "supports_ddl_extraction", + # The profiles of one table and the samples of one of its columns. The + # dialect answers these by aggregating the rows themselves, so every figure + # is exact and paid for in time. "column-profiles": "supports_column_data_profile", "value-samples": "supports_column_value_sample", "join-coverage": "supports_join_coverage", @@ -137,9 +137,8 @@ "indexes", "foreign-keys", "table-storage-metrics", - # The one-object families: the DDL that recreates one table, the profiles - # of its columns, and the samples of one of them. - "ddl", + # The one-object families: the profiles of one table's columns and the + # samples of one of them. "column-profiles", "value-samples", # ``join-coverage`` asks about two tables: the left one is NAME, the @@ -328,9 +327,8 @@ def create_parser(subparsers): %(prog)s dependencies --account myaccount %(prog)s dependencies orders --account myaccount - # The DDL that recreates one table, the exact per-column profiles, and the - # values one column holds. The profile reads every row, so it is paid for. - %(prog)s ddl orders --account myaccount + # The exact per-column profiles, and the values one column holds. The + # profile reads every row, so it is paid for. %(prog)s column-profiles orders --account myaccount %(prog)s value-samples orders --column status --account myaccount @@ -380,7 +378,7 @@ def create_parser(subparsers): default=None, help=( "Object name: a table for columns, indexes, foreign-keys, " - "table-storage-metrics, ddl, column-profiles, value-samples and " + "table-storage-metrics, column-profiles, value-samples and " "join-coverage (required); the object itself for views and " "dependencies (optional); refused for tables, database, " "databases, schemas, warehouses, routines, sequences, stages, " @@ -483,22 +481,6 @@ def _serialize_for_output(obj: Any) -> Any: return str(obj) -def _display_ddl(provider, statement, args) -> None: - """Show a DDL statement in the form that stays readable. - - A ``CREATE TABLE`` is text: a one-row table folds it into a cell that - wraps at the terminal width and interleaves with the other columns. Table - output therefore prints the statement itself, while the structured - formats keep the whole row so JSON and CSV still carry ``object_type`` - and ``native`` beside the text. - """ - if args.output == "table": - print(statement.ddl or "") - return - data = _serialize_for_output([statement]) - provider.display_results(data, title=f"DDL of {statement.object_name}") - - #: Said before a data-profile listing. It goes to stderr so the machine @@ -691,20 +673,8 @@ def _handle_introspect_sync(args, backend: SnowflakeBackend, provider): provider.display_results(data, title=title) # ---- the one-object families -------------------------------------- - # The DDL that recreates a table, and the profiles of one table or one - # column of it: the name is the table and the column is named on its - # own flag. - - elif args.type == "ddl": - statement = introspector.get_ddl_statement(args.name, object_type=None, schema=None) - if statement is None: - # Not "no such table": the object exists, and Snowflake - # answers with no text rather than an assembled fragment that - # would read as authoritative. The two are worth telling - # apart, so this exits without printing an empty row. - print(f"Error: no DDL available for '{args.name}'", file=sys.stderr) - sys.exit(1) - _display_ddl(provider, statement, args) + # The profiles of one table or one column of it: the name is the table + # and the column is named on its own flag. elif args.type == "column-profiles": # The note goes to stderr so the machine formats stay parseable, @@ -925,15 +895,6 @@ async def _handle_introspect_async(args, backend: AsyncSnowflakeBackend, provide title = f"Dependencies of {args.name}" if args.name else "Object Dependencies" provider.display_results(data, title=title) - elif args.type == "ddl": - statement = await introspector.get_ddl_statement( - args.name, object_type=None, schema=None - ) - if statement is None: - print(f"Error: no DDL available for '{args.name}'", file=sys.stderr) - sys.exit(1) - _display_ddl(provider, statement, args) - elif args.type == "column-profiles": print( "Note: column profiles are aggregated from the rows themselves " diff --git a/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_cli_introspect.py b/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_cli_introspect.py index 16d620b..a8d27fa 100644 --- a/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_cli_introspect.py +++ b/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_cli_introspect.py @@ -287,6 +287,13 @@ def test_a_type_that_is_not_listed_is_refused(self): with pytest.raises(SystemExit): build_parser().parse_args(["introspect", "table", "customers", "--account", "acct"]) + def test_ddl_is_not_a_type(self, capsys): + """The backend declines DDL extraction, so the CLI does not advertise it.""" + with pytest.raises(SystemExit) as excinfo: + build_parser().parse_args(["introspect", "ddl", "orders", "--account", "acct"]) + assert excinfo.value.code == 2 + assert "invalid choice" in capsys.readouterr().err + def test_help_lists_every_type_and_no_removed_aggregate(self, capsys): with pytest.raises(SystemExit): build_parser().parse_args(["introspect", "--help"]) @@ -294,6 +301,7 @@ def test_help_lists_every_type_and_no_removed_aggregate(self, capsys): for itype in cli_introspect.INTROSPECT_TYPES: assert itype in out assert "table users" not in out + assert "ddl" not in out def test_connection_arguments_flow_through(self): args = build_parser().parse_args( diff --git a/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_families_cli.py b/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_families_cli.py index 1b21573..2a28cc7 100644 --- a/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_families_cli.py +++ b/tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_families_cli.py @@ -1,5 +1,5 @@ # tests/rhosocial/activerecord_snowflake_test/feature/backend/cli/test_introspect_families_cli.py -"""The nine families the dialect answers, as the CLI asks for them. +"""The eight families the dialect declares, as the CLI asks for them. No server is needed here. What is checked is the wiring the CLI owns: that each type reaches its introspector method, that NAME and the type-specific @@ -11,10 +11,10 @@ methods, or calls the right one with a dropped argument, fails here rather than showing up as a plausible-looking listing. -Six of the nine are families this dialect cannot answer -- the switch is +Five of the eight are families this dialect cannot answer -- the switch is inherited as ``False`` -- so the gate is what the run meets, and the tests hold both sides of it: the refusal before anything connects, and the branch -that does run once a stub dialect answers ``True``. Six families answering +that does run once a stub dialect answers ``True``. Five families answering ``False`` is the honest state of this backend, and a CLI that runs them anyway would render a statement on a catalogue Snowflake does not have. @@ -34,13 +34,12 @@ from rhosocial.activerecord.backend.introspection.types import ( ColumnDataProfile, ColumnValueFrequency, - DDLStatement, JoinCoverage, ) -#: The nine families this change wires, and the switch that gates each. The -#: dialect declares the three data families and inherits the other six as -#: ``False``, so the three are asked and the six are refused -- both of which +#: The eight families this change wires, and the switch that gates each. The +#: dialect declares the three data families and inherits the other five as +#: ``False``, so the three are asked and the five are refused -- both of which #: the CLI must be able to do. NEW_FAMILIES = { "unused-indexes": "supports_unused_indexes_detection", @@ -48,7 +47,6 @@ "runtime": "supports_runtime_stats", "index-stats": "supports_index_stats", "dependencies": "supports_object_dependencies", - "ddl": "supports_ddl_extraction", "column-profiles": "supports_column_data_profile", "value-samples": "supports_column_value_sample", "join-coverage": "supports_join_coverage", @@ -71,7 +69,6 @@ "runtime", "index-stats", "dependencies", - "ddl", ) #: The families a real dialect answers, so their branch is reachable live. @@ -305,49 +302,6 @@ def test_an_unnamed_run_asks_for_the_whole_graph(self, monkeypatch): ] -class TestTheDdlBranch: - """The schema object, not a name, is what a caller parses.""" - - def _statement(self) -> DDLStatement: - return DDLStatement( - schema=None, - object_type="TABLE", - object_name="orders", - ddl="CREATE TABLE ORDERS (ID NUMBER)", - native=True, - ) - - def test_the_table_is_asked_for_by_name(self, monkeypatch): - introspector = _RecordingIntrospector({"get_ddl_statement": self._statement()}) - - _run_sync(monkeypatch, introspector, _name_required_argv("ddl")) - - assert introspector.calls == [ - ("get_ddl_statement", ("orders",), {"object_type": None, "schema": None}) - ] - - def test_the_engine_text_reaches_json_with_its_native_flag(self, monkeypatch, capsys): - introspector = _RecordingIntrospector({"get_ddl_statement": self._statement()}) - - _run_with_real_provider( - monkeypatch, introspector, _name_required_argv("ddl", ["-o", "json"]) - ) - - payload = json.loads(capsys.readouterr().out) - assert payload[0]["object_name"] == "orders" - assert payload[0]["ddl"] == "CREATE TABLE ORDERS (ID NUMBER)" - assert payload[0]["native"] is True - - def test_no_ddl_is_an_error_not_an_empty_listing(self, monkeypatch, capsys): - introspector = _RecordingIntrospector({"get_ddl_statement": None}) - - with pytest.raises(SystemExit) as excinfo: - _run_sync(monkeypatch, introspector, _name_required_argv("ddl")) - - assert excinfo.value.code == 1 - assert "no DDL available" in capsys.readouterr().err - - class TestTheColumnProfilesBranch: """The aggregate label must reach the reader; the numbers are not a catalogue.""" @@ -544,7 +498,7 @@ class TestTheArgumentRules: """The rules the CLI owns: a NAME and a flag are read, refused or required.""" @pytest.mark.parametrize( - "type_", ["ddl", "column-profiles", "value-samples", "join-coverage"] + "type_", ["column-profiles", "value-samples", "join-coverage"] ) def test_a_missing_table_name_exits_one(self, type_, capsys, monkeypatch): with pytest.raises(SystemExit) as excinfo: @@ -611,7 +565,7 @@ def test_the_stray_flag_is_refused_before_the_connection(self, monkeypatch): class TestTheCapabilityGate: """The switch is what the run consults, and it is consulted before the work. - Six of the nine families are ones this dialect cannot answer, and the + Five of the eight families are ones this dialect cannot answer, and the earlier behaviour was to run the branch anyway -- which produced an empty listing that read as "this database has none". Refusing first is what makes the two answers distinguishable, so the gate is asserted against the live @@ -670,7 +624,7 @@ def test_the_gate_never_reaches_the_introspector(self, type_, monkeypatch): introspector = _RecordingIntrospector() argv = ["introspect", type_] + CONNECTION - if type_ in ("ddl", "column-profiles", "value-samples", "join-coverage"): + if type_ in ("column-profiles", "value-samples", "join-coverage"): argv += ["orders"] if type_ in ("value-samples", "join-coverage"): argv += ["--column", "BUYER_ID"] @@ -735,12 +689,6 @@ class TestTheAsyncPath: ["posts"], ((), {"object_name": "posts", "schema": None}), ), - ( - "ddl", - "get_ddl_statement", - ["orders"], - (("orders",), {"object_type": None, "schema": None}), - ), ( "column-profiles", "list_column_data_profiles", @@ -773,16 +721,8 @@ def test_the_async_branch_calls_the_same_method_with_the_same_arguments( self, type_, method, extra, expected, monkeypatch ): # The one family whose answer is a single row rather than a listing - # needs an actual statement, because the branch reads a field off it. + # needs an actual row, because the branch reads a field off it. results = {method: []} - if method == "get_ddl_statement": - results[method] = DDLStatement( - schema=None, - object_type="TABLE", - object_name="orders", - ddl="CREATE TABLE ORDERS (ID NUMBER)", - native=True, - ) if method == "estimate_join_coverage": results[method] = JoinCoverage( left_table="orders", From e518a2fdcd5719d337b8998d739f49cc36298133 Mon Sep 17 00:00:00 2001 From: Zhao vistart Date: Mon, 9 Nov 2026 19:40:47 +0800 Subject: [PATCH 4/4] chore: add changelog fragment for PR #10 --- changelog.d/10.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/10.added.md diff --git a/changelog.d/10.added.md b/changelog.d/10.added.md new file mode 100644 index 0000000..ffe7482 --- /dev/null +++ b/changelog.d/10.added.md @@ -0,0 +1 @@ +Added Snowflake routines, sequences, stages, file formats, pipes, tasks, streams and identity/virtual-column introspection.