diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e7ff07f..2f105041 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,16 +52,16 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install activerecord and testsuite from PyPI + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Install activerecord and testsuite from PyPI - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - - name: Create test database run: | python -c " @@ -142,16 +142,16 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install activerecord and testsuite from PyPI + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Install activerecord and testsuite from PyPI - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - - name: Create test database run: | python -c " @@ -232,16 +232,16 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install activerecord and testsuite from PyPI + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Install activerecord and testsuite from PyPI - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - - name: Create test database run: | python -c " @@ -322,16 +322,16 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install activerecord and testsuite from PyPI + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Install activerecord and testsuite from PyPI - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - - name: Create test database run: | python -c " @@ -412,16 +412,16 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install activerecord and testsuite from PyPI + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Install activerecord and testsuite from PyPI - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - - name: Create test database run: | python -c " @@ -502,16 +502,16 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install activerecord and testsuite from PyPI + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Install activerecord and testsuite from PyPI - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - - name: Create test database run: | python -c " @@ -593,16 +593,16 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install activerecord and testsuite from PyPI + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Install activerecord and testsuite from PyPI - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - - name: Create test database run: | python -c " @@ -683,16 +683,16 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install activerecord and testsuite from PyPI + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Install activerecord and testsuite from PyPI - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - - name: Create test database run: | python -c " @@ -782,16 +782,16 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install activerecord and testsuite from PyPI + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Install activerecord and testsuite from PyPI - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - - name: Create test database run: | python -c " diff --git a/README.md b/README.md index fdf67024..38988455 100644 --- a/README.md +++ b/README.md @@ -208,8 +208,11 @@ PYTHONPATH=tests .venv3.14-ubuntu26.04/bin/pytest \ ## Documentation -This repository has no `docs/` tree. Backend notes live in -[.claude/architecture.md](.claude/architecture.md) and +[Database introspection](docs/introspection.md) states what this backend +answers, where each answer comes from, and the version floors that decide +whether it can be asked at all. + +Backend notes live in [.claude/architecture.md](.claude/architecture.md) and [.claude/testing.md](.claude/testing.md); core library documentation is at [python-activerecord/docs](https://github.com/rhosocial/python-activerecord/tree/main/docs). diff --git a/changelog.d/10.added.md b/changelog.d/10.added.md new file mode 100644 index 00000000..07c58ad0 --- /dev/null +++ b/changelog.d/10.added.md @@ -0,0 +1 @@ +Answer per-table storage from `sys.dm_db_partition_stats` with three scalar subqueries -- total, heap/clustered and nonclustered -- multiplied by the fixed 8 KiB page, filling `size_bytes`, `data_size_bytes` and `index_size_bytes` while a view keeps its row with `None` slots; `supports_table_storage_introspection()` declares the family and `row_count` stays `None` because an exact count is a scan and the listing reads no estimate. `SQLServerSequenceInfo` keeps its exact `Decimal`s and `is_cached` and gains `to_core()`, projecting name drift (`current_value` to `last_value`, `is_cycling` to `cycle`) and leaving non-integral figures `None` rather than widening the minimum. Four operational families answer -- unused indexes, index counters, runtime metrics and dependency edges -- while `table-stats` and `ddl` are refused with exit 2 because the DMV is keyed by object and index and no per-object rebuild text exists. Profiles, samples and join coverage render exact aggregations; nine CLI types derive from `_TYPE_CAPABILITIES`; and the 2005/2008/2012 availability floors are asked of the version accessor instead of being left to syntax errors. diff --git a/changelog.d/10.removed.md b/changelog.d/10.removed.md new file mode 100644 index 00000000..2ed379fa --- /dev/null +++ b/changelog.d/10.removed.md @@ -0,0 +1 @@ +Rename `SyncSQLServerIntrospector` to `SQLServerIntrospector` and `SyncSQLServerStatusIntrospector` to `SQLServerStatusIntrospector`, on core's `AbstractIntrospector`, `AbstractStatusIntrospector` and `IntrospectorExecutor`. Sync and async remain separate classes. diff --git a/docs/introspection.md b/docs/introspection.md new file mode 100644 index 00000000..1ec6710a --- /dev/null +++ b/docs/introspection.md @@ -0,0 +1,323 @@ +# Database Introspection + +The SQL Server backend answers three layers of questions: what the database declares (from `sys` catalog views and `INFORMATION_SCHEMA`), what it has been doing (four operational families out of six), and what its rows hold (three data families). `backend.introspector` is the entry point for all three, and every answer on this page is SQL Server's own: what this engine says, not the shape a cross-dialect consumer can expect. + +The contract those answers fill is core's, and this page does not restate it: the probe rules live in `backend/introspection/capabilities.md`, the operational shapes in `operations.md`, the profile and coverage types in `data-profiles.md`, all under [python-activerecord/docs](https://github.com/rhosocial/python-activerecord/tree/main/docs). What follows is only where SQL Server stands against that contract, and where its answers come from. + +## Overview + +- **Database information**: name, product version and collation, asked of the engine rather than read from a catalogue +- **Tables**: name, object type, `MS_Description` as the comment, and data/index bytes from `sys.dm_db_partition_stats` +- **Columns**: declared type, nullability, default, identity, character and numeric parameters, `MS_Description` +- **Indexes**: name, uniqueness, primary-key flag, type and column order, from `sys.indexes` and `sys.index_columns` +- **Foreign keys**: resolved column and table names, update/delete actions +- **Views and triggers**: definitions, updatability, firing events, `INSTEAD OF` flag +- **This backend's own families**: schemas, extended properties, procedures, functions and sequences, plus the primary-key read +- **Operational answers**: unused indexes, per-index counters, instance counters and by-name dependency edges; two families refused +- **Data answers**: per-column profiles, value listings and join coverage, all aggregated over the rows + +## What SQL Server Answers + +Every question in the introspection contract has a probe on the dialect, and a probe is a question about the server in front of you rather than about the product. This table is SQL Server's answer to all twenty-one. Read the `False` rows first: a probe that answers `False` says the question cannot be asked of this server, never that the answer was empty or zero. + +| Probe | SQL Server answers | The question it gates | +|-------|--------------------|-----------------------| +| `supports_introspection()` | `True` | Anything about the schema at all | +| `supports_database_info()` | `True` | The database's name, version and collation | +| `supports_table_introspection()` | `True` | Tables, views and system tables | +| `supports_column_introspection()` | `True` | Columns, identity and descriptions | +| `supports_index_introspection()` | `True` | Indexes, their columns and order | +| `supports_foreign_key_introspection()` | `True` | Foreign keys and their actions | +| `supports_view_introspection()` | `True` | Views and their definitions | +| `supports_trigger_introspection()` | `True` | Triggers, events and timing | +| `supports_table_storage_introspection()` | `True` | Per-table data and index bytes | +| `supports_runtime_stats()` | `True` from 2005 | What the instance says it is counting | +| `supports_table_stats()` | `False` on every version | Per-table usage counters | +| `supports_index_stats()` | `True` from 2005 | Per-index usage counters | +| `supports_unused_indexes_detection()` | `True` from 2005 | Indexes with no recorded read | +| `supports_object_dependencies()` | `True` from 2008 | The dependency graph — by-name references | +| `supports_ddl_extraction()` | `False` on every version | The text that recreates an object | +| `supports_ddl_extraction_native()` | `False` (inherited) | Whether that text is the engine's own | +| `supports_column_data_profile()` | `True` | What each column of a table holds | +| `supports_column_value_sample()` | `True` from 2012 | Which values occur, and how often | +| `supports_join_coverage()` | `True` | How much two columns' domains overlap | +| `supports_partition_info()` | `False` | Core's partition family | +| `supports_extensions()` | `False` | Installed add-on components | + +Five families beyond core's shared set are this backend's own, and each is declared in the mixin that owns it: `supports_extended_property_introspection()` for the `extended-properties` family over `sys.extended_properties`, plus `supports_schema_introspection()`, `supports_procedure_introspection()`, `supports_function_introspection()` and `supports_sequence_introspection()`. `get_primary_key()` is core's method, answered from `list_indexes()`'s `is_primary` flag; the primary-key statement is also what `list_columns()` runs first, so each column's `is_primary_key` is filled. `get_supported_introspection_scopes()` returns database, table, column, index, foreign key, view, trigger and sequence — core's `SCHEMA` scope is not among them, and extended properties and routines have no scope of their own. + +A `True` promises only that the query renders and the server accepts it; it says nothing about the shape of the answer or its price, and the price of every family is stated where the family is described. + +`supports_table_stats()` is `False` on every release, and the reason is the shape of the engine's counters rather than a missing catalogue. `sys.dm_db_index_usage_stats` is keyed by object **and index**: one row per index, never one row per table. Summing its rows per table would be an aggregate wearing the shape of a measured counter — a caller asking "how many scans did this table take" would receive a number no engine ever counted. Two of the slots the family names are not tracked at all: sequential scans are not distinguishable from indexed ones at the table level, and inserts and deletes are recorded nowhere — `user_updates` is the only write shape the DMV carries, and it counts writes of every kind. The row estimate is a different source again, and the table listing does not answer it either: `row_count` stays `None`, because an exact count would be a scan of the table and the listing reads no estimate. + +`supports_ddl_extraction()` is `False` on every release. The engine has no per-object retrieval call of the kind `SHOW CREATE TABLE` is: `OBJECT_DEFINITION` and `sys.sql_modules` hold the *body* of a routine, a view or a trigger — text the engine stores because it executes it — while a table's columns, constraints, indexes and storage are recorded as rows across `sys.columns`, `sys.indexes`, `sys.default_constraints` and more. Reassembling a `CREATE TABLE` from them is a reconstruction, and this family's rule is that a reconstruction is not an answer: `DDLStatement.native` exists to say which of the two a caller is looking at, and an assembly that used it would be claiming the engine said something it did not. `supports_ddl_extraction_native()` is `False` by inheritance for the same reason. + +`supports_unused_indexes_detection()`, `supports_index_stats()` and `supports_runtime_stats()` are the 2005 (9.0) floor: the three dynamic management views arrived there. `supports_object_dependencies()` is the 2008 (10.0) floor, where `sys.sql_expression_dependencies` arrived. `supports_column_value_sample()` is the 2012 (11.0) floor, because its cap rides `OFFSET 0 ROWS FETCH NEXT n ROWS ONLY`, which the engine parses only from that release — asking the version accessor is a truer answer than a statement that fails to parse. + +`supports_partition_info()` and `supports_extensions()` stay `False` by inheritance. Core's partition family has no method on this backend to oblige, and no catalogue is mapped for installable add-on components, so there is no answer behind a `True`. + +## Basic Usage + +```python +from rhosocial.activerecord.backend.impl.sqlserver.backend import SQLServerBackend + +backend = SQLServerBackend( + host="localhost", + port=1433, + database="mydb", + username="sa", + password="YourStrong!Passw0rd", + driver="{ODBC Driver 18 for SQL Server}", +) +backend.connect() +backend.introspect_and_adapt() + +introspector = backend.introspector + +info = introspector.get_database_info() +print(info.name, info.version, info.collation) + +for table in introspector.list_tables(): + print(table.name, table.table_type.value, + table.data_size_bytes, table.index_size_bytes, table.comment) + +for column in introspector.list_columns("orders"): + print(column.name, column.data_type_full, column.nullable.value) +``` + +`backend.introspect_and_adapt()` is the step that makes a version-gated probe answer for the server the handshake reached, rather than for the version the backend was constructed with. The CLI calls it at the entrance; a caller gating on a probe should do the same. + +Schema is a real level here. Every statement that takes one falls back to `dbo` when the caller names none, and the CLI's `schemas` type lists the database's schemas with their owners (`sys.schemas` left-joined to `sys.database_principals` on `principal_id`, because a schema and its owner need not share a name). The backend's own families sit beside the core API: + +```python +for schema in introspector.list_schemas(): + print(schema.name, schema.owner) + +for property_ in introspector.list_extended_properties( + class_name="OBJECT_OR_COLUMN", table_name="orders" +): + print(property_.name, property_.value) + +for routine in introspector.list_functions(): + print(routine.name, routine.type_code) +``` + +`list_extended_properties()` takes either a `major_id` it already has or a `table_name` to resolve — the named form joins `sys.tables` and `sys.schemas`, so a caller needs no first round trip to turn a name into an id. Its `class_name` is checked against the `class_desc` values the engine documents (`OBJECT_OR_COLUMN`, `INDEX`, `DATABASE`, `PARAMETER` and so on), and a schema filter is refused for a class whose `major_id` lives in another id space, because "no rows" there would be a silent contradiction rather than an answer. `value` is `sql_variant` and the driver cannot fetch it, so it is cast to `NVARCHAR(4000)` in the statement. + +## Structure answers worth knowing + +- **Database information is asked, not read.** The statement has no `FROM`: `DB_NAME()` and the `SERVERPROPERTY` values `ProductVersion`, `Collation` and `Edition`, each property cast to `NVARCHAR(128)`. The record carries the name, the version and the collation; `Edition` is projected but has no slot in the record, and `encoding` stays `None` — this engine reports a collation, not an encoding. +- **Tables come from `sys.objects`, not `INFORMATION_SCHEMA.TABLES`.** The information-schema view folds object kinds into two-word spellings and omits every `is_ms_shipped` row, so the one row that says "this is a system table" never reaches the parser. `sys.objects.type` (`U` user table, `V` view, `S` system table) and `is_ms_shipped` are the two facts the question is about, and `--include-system` changes the answer rather than being ignored. +- **No row count on the listing.** `IntrospectionTableInfo.row_count` stays `None`: an exact count would be a scan, and the listing reads no estimate. The storage slots beside it are measured, not estimated, and are `None` for a view rather than `0`. +- **Comments.** SQL Server has no `COMMENT` clause; `MS_Description` in `sys.extended_properties` is the only place a table's or column's comment lives. The table listing joins it with a `LEFT JOIN` on the object-level property alone (`minor_id = 0`), so a table without a comment keeps its row and a column description cannot be mistaken for the table's. The column listing reaches it through `sys.columns`, because the property is keyed by the object id and the column id, neither of which `INFORMATION_SCHEMA.COLUMNS` carries. +- **Storage bytes.** `sys.dm_db_partition_stats` counts one row per partition of one index: the partitions of `index_id` 0 (a heap) or 1 (a clustered index) are the table's own storage, everything above 1 is a nonclustered index's, and all partitions together are the total. The listing carries three scalar subqueries rather than a join, because a `GROUP BY` over that view would fold the object kinds and a view — which has no partition rows — would vanish from the listing instead of answering `None` for its storage. Page size is 8 KiB with no configuration switch, so the multiplication by 8192 is a fact of the engine. +- **Identity.** `INFORMATION_SCHEMA.COLUMNS` reports no identity flag; the column listing joins `sys.columns` and asks `COLUMNPROPERTY(..., 'IsIdentity')`. `DATETIME_PRECISION` is projected because it is the only place the fractional-seconds precision of `time`/`datetime2`/`datetimeoffset` lives. +- **Indexes.** `sys.indexes` joined to `sys.index_columns` and `sys.columns`; a heap has no named index row and is excluded, so `index_id` 0 never appears as an index. +- **Triggers.** Firing events are not a column of `sys.triggers`; they live one table over, in `sys.trigger_events`, one row per event. The join is a `LEFT JOIN`, one row per event, and the parser folds contiguous rows back into one trigger — so a trigger with no event row is still a trigger, and its definition comes from `OBJECT_DEFINITION`. +- **Views.** `INFORMATION_SCHEMA.VIEWS` carries `VIEW_DEFINITION`, which is `NULL` for a view whose text the caller may not read — unlike `OBJECT_DEFINITION()`, which returns the text to a caller holding `VIEW DEFINITION`. Both the list and the single-view question read that one view, so they cannot disagree, and `include_system` is not consulted because the view holds no server-owned row. + +## Operational Families + +SQL Server implements four of the six. + +| Family | Method | SQL Server source | Price | +|--------|--------|-------------------|-------| +| Unused indexes | `list_unused_indexes()` | `sys.dm_db_index_usage_stats` through `sys.objects`/`sys.schemas`/`sys.indexes` | One DMV read | +| Table counters | `list_table_usage_stats()` | none — refused by the probe | no query is run | +| Runtime metrics | `list_runtime_metrics()` | `sys.dm_os_performance_counters` | One instance-wide read | +| Index counters | `list_index_usage_stats()` | the same index usage DMV | One DMV read | +| Dependencies | `list_object_dependencies()` | `sys.sql_expression_dependencies` | One catalogue read | +| DDL | `get_ddl_statement()` | none — refused by the probe | no query is run | + +Every counter here is since the view was last flushed, which is the engine's own window and not a resettable epoch: the index usage view flushes on its own schedule after a restart, and the performance counters start again when the instance does. Nothing in the answer dates the window, so a zero is "none in the stretch the engine remembers", never "never". That is also why a cached answer ages faster here than in the declaration layer: a schema cached yesterday is probably still the schema, while the window a counter described has kept running. + +### Unused Indexes + +One row per index of the schema, addressed by three names the DMV does not itself carry: `sys.dm_db_index_usage_stats` addresses everything by `database_id`, `object_id` and `index_id`, so it is joined through `sys.objects` and `sys.schemas` for the table and schema names, and through `sys.indexes` for the index name. Two predicates are the answer to "unused" as this engine means it: `index_id > 0` excludes a heap's row, which is not an index, and the sum of the three read counters — `user_seeks`, `user_scans`, `user_lookups` — being zero says no query the engine counted has read it. + +The answer is "worth reviewing", never "safe to drop". An index absent from the listing is absence of evidence, not evidence of use, and the DMV only reports the indexes it knows about. `DBCC SQLPERF('sys.dm_os_wait_stats', CLEAR)` clears some counters, but this view keeps its own schedule. Reading it needs `VIEW DATABASE STATE` (or `VIEW SERVER STATE`); without that grant the view returns only the objects the caller owns, so a caller with less privilege gets a shorter list rather than an error, and the probe cannot say so because it can ask a version, not a grant. + +### Index Usage Counters + +The same DMV, one slot further. `scans` is the sum of the same three read shapes — that is not a derived measurement but the same counter spelled in three parts — and `user_updates` rides the row into `extra` under the name `writes`, because an update is not a scan and the two are not interchangeable. `rows_read` and `rows_fetched` stay `None`: the view keeps no per-index row counter, and an operation is not a row. `size_bytes` stays `None` too; the DMV does not carry it. Unlike the unused listing, a zero here is a measurement, and the difference between one scan and a million is what the family exists to show. + +### Runtime Metrics + +`sys.dm_os_performance_counters` names what is actually being maintained on this instance, one row per counter grouped by the object that owns it. `counter_name` is projected as `name`, `cntr_value` as `value`, `cntr_type` as `kind` — the engine's own classification, so a caller can group without learning the DMV's vocabulary — and `object_name` as `source`, which is the counter group that owns it (`SQLServer:Buffer Manager` and friends). `instance_name`, the per-node or per-LRU split that makes two rows of the same counter distinguishable, travels in `extra`. + +There is no schema filter, and that is deliberate: the counters describe a running instance and carry no schema column, so a filter here would refuse a correct query for a nonsensical reason. `unit` and `description` stay `None`. Reading the view needs `VIEW SERVER STATE`; without that grant it answers nothing, and a listing that silently loses every row there is a privilege statement rather than a measurement. + +### Object Dependencies + +`sys.sql_expression_dependencies` records, per reference, which object depends on which *named* entity. The dependent side is the referencing object — the statement joins that view to `sys.objects` on `referencing_id` and to `sys.schemas` for the schema — and the referenced side is a *name*, `referenced_entity_name`, never an id, which is why the join is on the referencing object and a second join would have to resolve the name. `type_desc` of the referencing object arrives as `dependent_type`, and `referenced_class_desc` arrives as `kind` — `OBJECT`, `COLUMN`, `TYPE` and so on. + +`action` and `referenced_type` stay `None`: this view records no referential action, and no type for the referenced entity beyond the class descriptor. It is a real graph and a partial one — a foreign key is not a row here, and an object bound through dynamic SQL is invisible to it. A reference that leaves the database is not filtered out: the row still names the entity, and its server or database qualifier is not in the projection. Naming an object narrows the answer to its dependants; omitting the name returns the whole graph for the schema. + +### The two refusals + +`table-stats` and `ddl` are refused by their probes on every release, and the CLI exits **2** with "not supported by this server" without sending a statement. That is a different answer from an empty list: "this server cannot be asked" is not "the table was never read", and an assembled `CREATE TABLE` would be a statement the engine never made. The roundtrip matrix in this repository keeps an explicit pin for `DDLStatementQueryStatement` with this reason. + +Every family — answered or refused — is reached through the same introspector, sync and async alike: + +```python +backend.introspect_and_adapt() + +if backend.dialect.supports_unused_indexes_detection(): + for index in backend.introspector.list_unused_indexes(): + print(f"{index.table_name}.{index.index_name}") + +for edge in backend.introspector.list_object_dependencies("orders"): + print(f"{edge.dependent} -> {edge.referenced} ({edge.kind})") +``` + +## Data Families + +The three data families measure the rows rather than read a catalogue. SQL Server publishes statistics, and they are the reason `column-profiles` is worth a paragraph rather than a sentence: `sys.stats` names a *statistics object*, `sys.stats_columns` maps that object to one or more columns, and `sys.dm_db_stats_properties` reports what the last update of that object measured — `rows`, `rows_sampled`, `steps`, `modification_counter`, `last_updated`, `persisted_sample_percent`. The key is the object, never the column. A single-column statistics object is a fair proxy for its column, but a two-column one describes the pair's leading prefix, and the same column can appear in several objects with different samples, so summing or picking one would invent a per-column fact the catalogue does not hold. + +What is missing is sharper than what is keyed oddly. There is no per-column null count anywhere in the catalogue — nullability is a *declaration* in `sys.columns`, never a measurement — and no per-column distinct count either. `sys.dm_db_stats_histogram` (2016+) reports `range_high_key`, `equal_rows` and `distinct_range_rows` per step of one object, which is a bucket summary for that object's leading column, not a count of the values in a column. The one figure the catalogue could contribute, `last_updated`, would be the vintage of a statistics object, not of the profile, and this family's rule is that a figure travels with the source and vintage that produced it. A profile that mixed a sampled estimate with exact counts would have to be labelled with one source it does not deserve. + +So all five figures come from one place, and it is the rows: exact, and paid for in time. + +| Family | Method | Price | +|--------|--------|-------| +| Column profiles | `list_column_data_profiles(table)` | One pass over the table per column | +| Value samples | `sample_column_values(table, column, limit=100)` | One grouped pass over the column, plus a sort | +| Join coverage | `estimate_join_coverage(left_table, left_column, right_table, right_column)` | Two sides read for their distinct domains, plus the intersection | + +The `aggregate` label is projected *into* each row rather than stamped onto the result afterwards, so a consumer comparing a SQL Server profile with another engine's estimate can tell them apart without knowing which engine it asked. + +### Column Profiles + +One `UNION ALL` arm per column of the table, and each arm reads the table once: the null count is a `SUM` over a `CASE`, the distinct count `COUNT(DISTINCT ...)`, the endpoints the engine's `MIN`/`MAX`, and the row count rides the same scan as the divisor. A table of ten columns is therefore read ten times — one round trip, ten scans. `null_fraction` is derived in the parser (`null_count / row_count`), so a row count that may be zero never has to be guarded in SQL, and `SUM` over no rows is NULL: an empty table answers "not measured" for the fraction rather than `0.0`, while `distinct_count` and `row_count` are `0`, which are measurements. + +The arms are rendered individually and joined by the set operator's word — the one place in this module where a set operator is written as text — because a column list known only at run time has no operand to represent it, and each arm remains a real statement with its own parameters, concatenated in arm order. An empty column list answers with no rows. + +Three type decisions are written into the projection rather than left to the reader: + +- **`COUNT_BIG` and `BIGINT`.** SQL Server's `SUM` and `COUNT` return `INT`, so a table with more than 2,147,483,647 rows — exactly the table worth profiling — would overflow the very counts the profile exists to report. The null flag is cast to `BIGINT` before it is summed, and the row count is `COUNT_BIG(*)`. +- **Endpoints cast to `NVARCHAR(MAX)`.** A `UNION ALL` resolves *one* result type per column position by data-type precedence and implicitly converts the other arms to it, so the `min_value` slot would have to hold an `int` from one arm and a `varchar` from another: `int` wins, and the text value is converted to a number, which fails. A `decimal` beside a `datetime` fails the same way in the other direction. Nothing in the formatter knows a column's declared type, so there is no per-type cast to choose; text is the one target every scalar column converts to. The figures are exact; the Python types are the union's, and a `datetime` endpoint renders without its sub-minute part because that is what SQL Server's style-free conversion produces. +- **The label is `NVARCHAR(128)`.** SQL Server's `CHAR` and `NVARCHAR` default to length 1 in a `CAST`, which would truncate the word `aggregate` to its first character. + +The column name is also bound rather than written into the statement, because a non-prefixed literal would be typed with the database's code page and an identifier outside that page would arrive as question marks; a parameter is typed Unicode by the driver. + +A table holding an `xml`, `text`/`ntext`/`image` or CLR-typed column cannot be profiled this way at all: SQL Server refuses `MIN` and `MAX` on those types, and the statement fails rather than answering a partial profile. That is the honest limit of an aggregate, and it is a limit of the *statement*, not of the capability probe. + +### Value Samples + +A `GROUP BY` over the column plus a sort of the groups — the same order of cost as the profile, which is why the two are separate calls. `ORDER BY COUNT_BIG(*) DESC` is what makes the answer "the most frequent values" rather than "the first values", and ties break by the value so two runs of the same query agree. `COUNT_BIG` rather than `COUNT` for the same overflow reason as the profile; `frequency` stays `None` because the engine reports a count and the denominator is the row count the profile carries. + +The cap rides `OFFSET 0 ROWS FETCH NEXT n ROWS ONLY`, which is why the probe has a 2012 floor. `n` is an integer the statement carries, so no caller text is spliced into the statement. Only `value` and `count` are projected. `GROUP BY` groups NULLs, so a column that contains NULL has an entry whose `value` is `None`, and it is usually the most frequent one — NULL is a member of the domain in this listing. + +### Join Coverage + +Both fractions come from one statement. For each side, the numerator counts the distinct values that also occur on the other side — a `COUNT_BIG(*)` over a derived `DISTINCT` — and the denominator counts that side's distinct non-NULL values. Distinct on both sides is what makes the figure a coverage of values rather than of rows: a key repeated a thousand times that matches once would otherwise read as well covered. NULLs are excluded from both halves explicitly; a `DISTINCT` would count NULL as one value and credit a match that can never join. `NULLIF(..., 0)` makes a column with nothing to match answer NULL rather than `0.0`, so "nothing matched" and "nothing to match" stay distinct. + +The numerator is multiplied by `1.0` before it is divided, and that is not decoration: SQL Server divides two integers with integer division, so `COUNT(*) / COUNT(DISTINCT ...)` would answer 0 or 1 and lose every fraction between them. The multiplier makes the division decimal, which is exact for these counts rather than approximate. Collations are not reconciled — the engine compares the two columns under whichever collation applies to the comparison, so values the database would treat as equal may be counted as distinct, and across mismatched collations the fractions are a *lower* bound, never an upper one. The price is the two sides read for their distinct domains plus the intersection, which is why this is a call of its own and never part of a profile. + +```python +profiles = backend.introspector.list_column_data_profiles("orders") +for profile in profiles: + print(profile.column_name, profile.null_fraction, profile.distinct_count, + profile.row_count, profile.source) + +samples = backend.introspector.sample_column_values("orders", "status", limit=20) +for sample in samples: + print(sample.value, sample.count) # value is None when NULL is a group + +coverage = backend.introspector.estimate_join_coverage( + "orders", "customer_id", + "customers", "id", +) +print(coverage.left_coverage, coverage.right_coverage) +``` + +## Version and Availability Floors + +Two limits decide whether an operational or data family can answer at all, and neither is a property of the query text. + +| Family | Floor | What a missing floor looks like | +|--------|-------|---------------------------------| +| `unused-indexes`, `index-stats`, `runtime` | SQL Server 2005 (version 9.0), where the DMVs arrive | probe `False`; the CLI exits 2 without running the query | +| `dependencies` | SQL Server 2008 (version 10.0), where `sys.sql_expression_dependencies` arrives | probe `False`; the CLI exits 2 | +| `value-samples` | SQL Server 2012 (version 11.0), where `OFFSET ... FETCH` parses | probe `False`; the CLI exits 2 | +| `table-stats`, `ddl` | none — no release answers them | probe `False` on every version; the CLI exits 2 | +| every other family | none established here | — | + +The floors are asked of the version accessor, not left to a runtime syntax error: a server below one of them is refused at the capability question with "cannot answer", which is a different and truer answer than a query that fails to parse. Every release this repository tests against — 2017, 2019, 2022 and 2025 — is well past all three, so the gates change nothing there and keep an older server from an execution error. + +A probe answering `False` is "the question cannot be asked here", not "nothing was used". It does not say that no index was read or that the database has no indexes: it says the server cannot be asked. `False` lets a caller take its fallback knowingly; the empty list is reserved for a question that was asked and had nothing to report. + +The probes do not test grants. No privilege floor is established here: a probe answers about the server and its settings, not about the account's privileges. Two of the DMVs need one — `VIEW DATABASE STATE` for the index usage view, `VIEW SERVER STATE` for the performance counters — and without it the engine answers a short list or nothing at all, not an error. That is an answer about what the caller may see, and the probe said `True` because the server can answer, not because this account may ask. + +## Command Line Introspection + +```bash +# Schemas and their owners, then the tables of the connected database +python -m rhosocial.activerecord.backend.impl.sqlserver introspect schemas --database mydb +python -m rhosocial.activerecord.backend.impl.sqlserver introspect tables --database mydb + +# One table's columns, and its MS_Description properties +python -m rhosocial.activerecord.backend.impl.sqlserver introspect columns orders --database mydb +python -m rhosocial.activerecord.backend.impl.sqlserver introspect extended-properties orders --database mydb + +# Operational readings +python -m rhosocial.activerecord.backend.impl.sqlserver introspect unused-indexes --database mydb +python -m rhosocial.activerecord.backend.impl.sqlserver introspect index-stats --database mydb +python -m rhosocial.activerecord.backend.impl.sqlserver introspect runtime --database mydb +python -m rhosocial.activerecord.backend.impl.sqlserver introspect dependencies orders --database mydb + +# Exact profiles over the rows +python -m rhosocial.activerecord.backend.impl.sqlserver introspect column-profiles orders --database mydb +python -m rhosocial.activerecord.backend.impl.sqlserver introspect value-samples orders \ + --column status --limit 20 --database mydb + +# The two families this engine does not answer are refused after connecting, exit 2 +python -m rhosocial.activerecord.backend.impl.sqlserver introspect table-stats --database mydb +``` + +### Introspection Types + +| Type | Description | Name | +|------|-------------|------| +| `tables` | User tables and views, plus system tables with `--include-system` | not accepted | +| `views` | List all views, or describe one | optional (view) | +| `columns` | Column information of one table | required (table) | +| `indexes` | Index information of one table | required (table) | +| `foreign-keys` | Foreign keys of one table | required (table) | +| `extended-properties` | `MS_Description` and friends on one table | required (table) | +| `triggers` | All triggers, or one table's | optional (table) | +| `database` | Name, product version and collation | not accepted | +| `procedures` | Stored procedures in `dbo` | not accepted | +| `functions` | User-defined functions in `dbo` | not accepted | +| `sequences` | Sequences in `dbo` | not accepted | +| `schemas` | Schemas and their owners | not accepted | +| `unused-indexes` | Indexes with no recorded read, from 2005 | not accepted | +| `table-stats` | Per-table usage counters — refused on every version | not accepted | +| `runtime` | Instance performance counters | not accepted | +| `index-stats` | Per-index usage counters, from 2005 | not accepted | +| `dependencies` | By-name dependency edges, from 2008 | optional (table) | +| `ddl` | The DDL that recreates one object — refused on every version | required (table) | +| `column-profiles` | What each column of a table holds | required (table) | +| `value-samples` | Most frequent values of one column | required (table), plus `--column` | +| `join-coverage` | Overlap of two columns' value domains | not accepted; four flags required | + +The nine operational and data families are the set the other CLIs ask for too, and each one names the probe that answers for it here — which is not automatically the same answer it gives elsewhere. `views NAME` is a *view* name and `triggers NAME` is a *table* name: the two are not the same kind of argument, which is why the rule table names both. A name the type cannot use is refused (exit 2) rather than ignored; a missing required name or flag is an incomplete invocation (exit 1); a flag the type does not read is refused (exit 2). A `views` name or an `extended-properties` name that answers nothing is reported as "not found" (exit 1), and for extended properties the table is first checked against the catalogue, because a table with no `MS_Description` and a table that is not there both answer with no rows. `ddl` and `join-coverage` answering `None` — "the engine will not produce a text", "the overlap could not be measured" — also leave with 1 rather than printing a row of nulls a reader would take for a measurement. + +`--limit` is forwarded to `sample_column_values()` only when it is given; without it the introspector's own default answers (`VALUE_SAMPLE_LIMIT`, 100), so the number is not written in two places. `--include-system` changes the table listing; the view listing reads `INFORMATION_SCHEMA.VIEWS`, which holds no server-owned row, so the flag has nothing to add there. `column-profiles` writes one note to stderr saying the profiles are aggregated from the rows themselves and paid for with a full scan per column, so a caller piping JSON to a consumer still sees where the numbers came from. + +### Output and connection + +`-o table` (default, rich), `json`, `csv` and `tsv` are available for every type; `--rich-ascii` switches the table borders. Connection parameters can be given as flags or as environment variables: + +| Flag | Environment variable | Default | +|------|----------------------|---------| +| `--host` | `SQLSERVER_HOST` | `localhost` | +| `--port` | `SQLSERVER_PORT` | `1433` | +| `--database` | `SQLSERVER_DATABASE` | required for introspection | +| `--user` / `--username` | `SQLSERVER_USERNAME` | `sa` | +| `--password` | `SQLSERVER_PASSWORD` | empty | +| `--driver` | `SQLSERVER_DRIVER` | `ODBC Driver 18 for SQL Server` | + +`--trusted-connection` switches to Windows authentication. `--ssl` selects one of `auto`, `require`, `verify-ca`, `verify-full` and `disabled`, and `--encrypt`/`--no-encrypt` plus `--trust-server-certificate`/`--no-trust-server-certificate` take precedence over it. + +A named connection profile (`--named-connection`) may stand in for `--database` when one is configured. `--async` runs the same command through `AsyncSQLServerBackend`; every family is asked the same way and the answers are the same. diff --git a/pyproject.toml b/pyproject.toml index daaf2255..a9b1843b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ classifiers = [ ] dependencies = [ "pyodbc>=5.3.0", - "rhosocial-activerecord>=1.0.0.dev0,<2.0.0", + "rhosocial-activerecord>=1.0.0.dev30,<2.0.0", ] [project.scripts] diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/backend/backend.py b/src/rhosocial/activerecord/backend/impl/sqlserver/backend/backend.py index 5a66644f..5b7bd378 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/backend/backend.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/backend/backend.py @@ -657,11 +657,11 @@ def _create_introspector(self): """Create a SQL Server introspector. Returns: - SyncSQLServerIntrospector instance + SQLServerIntrospector instance """ - from ..introspection import SyncSQLServerIntrospector - from rhosocial.activerecord.backend.introspection.executor import SyncIntrospectorExecutor - return SyncSQLServerIntrospector(self, SyncIntrospectorExecutor(self)) + from ..introspection import SQLServerIntrospector + from rhosocial.activerecord.backend.introspection.executor import IntrospectorExecutor + return SQLServerIntrospector(self, IntrospectorExecutor(self)) def _parse_explain_result(self, raw_rows, sql, duration): """Parse SQL Server EXPLAIN result. diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/info.py b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/info.py index 138dd668..848db971 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/info.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/info.py @@ -51,12 +51,10 @@ ) from .connection import add_connection_args, add_version_arg, resolve_connection_config_from_args -from .output import create_provider, RICH_AVAILABLE +from .output import INFO_OUTPUT_CHOICES, create_provider, RICH_AVAILABLE logger = logging.getLogger(__name__) -OUTPUT_CHOICES = ['table', 'json'] - DIALECT_SPECIFIC_GROUPS = {"SQL Server Native"} PROTOCOL_FAMILY_GROUPS: Dict[str, list] = { @@ -121,7 +119,7 @@ def create_parser(subparsers): parser.add_argument( '-o', '--output', - choices=OUTPUT_CHOICES, + choices=INFO_OUTPUT_CHOICES, default='table', help='Output format (default: table)', ) diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/introspect.py index d811bf42..8ce42271 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/introspect.py @@ -12,6 +12,20 @@ and nowhere else, so a new type is one entry there plus one title. The sync and async paths differ in exactly one keyword: whether the method is awaited. + +What NAME means for a type is decided once, in ``_NAME_RULE``, and applied by +one validator before the connection is opened. It is not decided in the +branches: a name a type has no question for is refused with exit 2 rather +than parsed and dropped, and a type added without a rule is a type nobody has +decided for. + +The nine families the MySQL and PostgreSQL CLIs ask for are offered here +too, each gated on the switch that answers for it. SQL Server answers seven +of the nine from its dynamic management views; two it does not answer at all +-- per-table usage counters, which it keeps per *index*, and DDL retrieval, +which the engine has no call for -- so those probes answer ``False`` and the +capability check refuses them with exit 2 rather than letting a rendered +statement fail at execution. """ import argparse @@ -24,9 +38,7 @@ from rhosocial.activerecord.backend.errors import ConnectionError, QueryError from rhosocial.activerecord.backend.impl.sqlserver.backend import SQLServerBackend from .connection import add_connection_args, resolve_connection_config_from_args -from .output import create_provider - -OUTPUT_CHOICES = ['table', 'json', 'csv', 'tsv'] +from .output import OUTPUT_CHOICES, create_provider #: CLI type -> the dialect capability switch that answers for it. #: @@ -49,6 +61,22 @@ "functions": "supports_function_introspection", "sequences": "supports_sequence_introspection", "schemas": "supports_schema_introspection", + # The nine families shared with the MySQL and PostgreSQL CLIs. Each names + # the probe that answers for it here, which is not automatically the same + # answer it gives there: the dependency graph and the runtime listing come + # from the DMVs, the unused-index and index-counter families from the + # index usage DMV, the three data families from the rows themselves, and + # two -- per-table usage counters and DDL retrieval -- are refused by the + # engine's own answer rather than by an absent CLI branch. + "unused-indexes": "supports_unused_indexes_detection", + "table-stats": "supports_table_stats", + "runtime": "supports_runtime_stats", + "index-stats": "supports_index_stats", + "dependencies": "supports_object_dependencies", + "ddl": "supports_ddl_extraction", + "column-profiles": "supports_column_data_profile", + "value-samples": "supports_column_value_sample", + "join-coverage": "supports_join_coverage", } INTROSPECT_TYPES = list(_TYPE_CAPABILITIES) @@ -63,7 +91,13 @@ #: #: ``views NAME`` is a view name and ``triggers NAME`` is a *table* name: the #: two are not the same kind of argument, which is why this table exists. -_NAME_RULES = { +#: +#: The order is the order ``_TYPE_CAPABILITIES`` declares and the parser and +#: ``--help`` print, and the two tables cover the same keys: a type offered +#: without a rule is a type whose NAME decision nobody has made. One validator +#: reads this table, before the connection is opened, and the dispatcher +#: consumes its answer rather than re-deciding per branch. +_NAME_RULE = { "tables": "forbidden", "views": "optional", "columns": "required", @@ -76,6 +110,46 @@ "functions": "forbidden", "sequences": "forbidden", "schemas": "forbidden", + # The nine shared families: the counters and the listings describe the + # database and take no name; the graph narrows to one object and answers + # the whole graph without one; the one-object families ask about a table. + "unused-indexes": "forbidden", + "table-stats": "forbidden", + "runtime": "forbidden", + "index-stats": "forbidden", + "dependencies": "optional", + "ddl": "required", + "column-profiles": "required", + "value-samples": "required", + "join-coverage": "forbidden", +} + +#: Options whose meaning depends on the type, and whether they must be given. +#: +#: ``--column`` and ``--limit`` belong to ``value-samples``, the four join +#: names to ``join-coverage``, and nothing else has an option beyond NAME. A +#: flag supplied to a type that does not read it is refused rather than +#: ignored, for the same reason a NAME nothing reads is: a silently dropped +#: argument produces a successful run that looks like it answered a narrower +#: question. +_TYPE_OPTIONS = { + "value-samples": ( + ("column", "--column", True), + ("limit", "--limit", False), + ), + "join-coverage": ( + ("left_table", "--left-table", True), + ("left_column", "--left-column", True), + ("right_table", "--right-table", True), + ("right_column", "--right-column", True), + ), +} + +#: Every option in ``_TYPE_OPTIONS``, by attribute, for the stray-flag check. +_TYPE_OPTION_FLAGS = { + attribute: flag + for options in _TYPE_OPTIONS.values() + for attribute, flag, _required in options } #: Titles for the one-result types. The table-scoped types build their title @@ -89,6 +163,13 @@ "list_sequences": "Sequences", "list_schemas": "Schemas", "get_database_info": "Database Info", + "list_unused_indexes": "Indexes with no recorded use", + "list_table_usage_stats": "Table usage counters", + "list_runtime_metrics": "Runtime metrics", + "list_index_usage_stats": "Index usage counters", + "list_object_dependencies": "Object dependencies", + "list_column_data_profiles": "Column profiles", + "estimate_join_coverage": "Join coverage", } @@ -138,6 +219,28 @@ def create_parser(subparsers): # List sequences %(prog)s sequences --database mydb + # Operational readings: indexes with no recorded use, per-index counters, + # live instance counters, the dependency graph + %(prog)s unused-indexes --database mydb + %(prog)s index-stats --database mydb + %(prog)s runtime --database mydb + %(prog)s dependencies --database mydb + %(prog)s dependencies orders --database mydb + + # Data profiles: exact counts, paid for with a full scan per column + %(prog)s column-profiles orders --database mydb + %(prog)s value-samples orders --column status --limit 20 --database mydb + + # How much two columns' value domains overlap + %(prog)s join-coverage --left-table orders --left-column customer_id \\ + --right-table customers --right-column id --database mydb + + # The two families SQL Server does not answer are refused by capability, + # not hidden: table-stats and ddl exit 2 with "not supported by this + # server" once the version is known. + %(prog)s table-stats --database mydb + %(prog)s ddl orders --database mydb + # Output as JSON %(prog)s tables --database mydb -o json @@ -148,8 +251,10 @@ def create_parser(subparsers): NAME per type: columns, indexes, foreign-keys, extended-properties a table name, required + ddl, column-profiles, value-samples + a table name, required views a view name, optional - triggers a table name, optional + triggers, dependencies optional narrowing everything else no name: refused with exit 2, never ignored """, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -189,27 +294,137 @@ def create_parser(subparsers): action="store_true", help="Include system tables in output", ) + parser.add_argument( + "--column", + default=None, + help="Column name for the value-samples type.", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help=( + "Maximum values to list for value-samples " + "(default: the introspector's own limit)." + ), + ) + parser.add_argument( + "--left-table", + dest="left_table", + default=None, + help="Left table for the join-coverage type.", + ) + parser.add_argument( + "--left-column", + dest="left_column", + default=None, + help="Left column for the join-coverage type.", + ) + parser.add_argument( + "--right-table", + dest="right_table", + default=None, + help="Right table for the join-coverage type.", + ) + parser.add_argument( + "--right-column", + dest="right_column", + default=None, + help="Right column for the join-coverage type.", + ) return parser -def _refuse_unused_name(args) -> None: - """Refuse a NAME the requested type cannot use, rather than ignore it. +def _validate_name(args) -> None: + """Apply the NAME rule table before anything connects. - ``introspect tables users`` used to list every table and exit 0, which - reads as "the filter matched everything" or "the filter was applied" -- - the most misleading shape a wrong answer can take. A name for a type that - answers for the database is a misuse of the argument, so it leaves with 2. + A *missing* name is an incomplete invocation and leaves with 1; a name + the type has no use for is a request this command declines and leaves with + 2. Both are checked here, once, rather than being left to each branch to + remember -- a branch that forgot would answer a wider question than the + one asked and look successful. """ - if not args.name: - return - if _NAME_RULES.get(args.type) != "forbidden": + rule = _NAME_RULE.get(args.type) + if rule == "required" and not args.name: + print( + f"Error: Table name is required for '{args.type}' introspection", + file=sys.stderr, + ) + sys.exit(1) + if rule == "forbidden" and args.name: + print( + f"Error: {args.type} introspection takes no name", + file=sys.stderr, + ) + sys.exit(2) + + +def _validate_options(args) -> None: + """Refuse a flag the type does not read, or a missing one it needs. + + A missing required option is an incomplete invocation (exit 1); an option + the type has no use for is a request this command declines (exit 2), like + a NAME it cannot use. Reading the flags with ``getattr`` keeps the + hand-built namespaces the offline tests use working: an absent attribute + is "not given", not an error. + """ + options = _TYPE_OPTIONS.get(args.type, ()) + accepted = {attribute for attribute, _flag, _required in options} + + missing = [ + flag + for attribute, flag, required in options + if required and not getattr(args, attribute, None) + ] + if missing: + print( + f"Error: {', '.join(missing)} is required for " + f"'{args.type}' introspection", + file=sys.stderr, + ) + sys.exit(1) + + for attribute, flag in _TYPE_OPTION_FLAGS.items(): + if attribute in accepted: + continue + if getattr(args, attribute, None) not in (None, ""): + print( + f"Error: '{args.type}' introspection takes no {flag}", + file=sys.stderr, + ) + sys.exit(2) + + +def _value_sample_arguments(args) -> Dict[str, int]: + """The optional arguments ``value-samples`` sends, given the parsed argument. + + ``limit`` is forwarded only when the caller supplied ``--limit``. When the + flag is absent the introspector's signature answers with its own default, + so a forwarder that filled in ``100`` would be a second place writing a + number the signature already owns -- and a place no test could keep in + step with it. + """ + limit = getattr(args, "limit", None) + return {} if limit is None else {"limit": limit} + + +def _display_ddl(args, provider, statement) -> None: + """Show one object's DDL as the engine handed it back. + + A DDL text is a block, not a field of a row: a bordered cell wraps every + one of its lines and reads worse than the text itself. So the terminal + format prints the engine's own text, and the formats a caller parses keep + the record -- schema, object type, object name, ``native`` and the text -- + which is what ``native`` is for: it says the engine retrieved this rather + than the library assembling it. + """ + record = _serialize_for_output(statement) + if args.output != "table": + provider.display_results([record], title=f"DDL of {args.name}") return - print( - f"Error: {args.type} introspection takes no name", - file=sys.stderr, - ) - sys.exit(2) + print(f"-- {record.get('object_type') or 'OBJECT'} {record.get('object_name') or args.name}") + print(record["ddl"]) def _require_capability(backend, kind: str) -> None: @@ -246,9 +461,10 @@ def handle(args): print("Error: --database is required for introspection", file=sys.stderr) sys.exit(1) - # Before a connection is opened: whether a NAME belongs to the type is a - # question about the request, not about the server. - _refuse_unused_name(args) + # Before a connection is opened: whether a NAME or a flag belongs to the + # type is a question about the request, not about the server. + _validate_name(args) + _validate_options(args) config = resolve_connection_config_from_args(args) @@ -266,9 +482,17 @@ def handle(args): # --------------------------------------------------------------------------- -def _require_name(args, kind: str) -> str: - """Return the object name a table-scoped type needs, or exit.""" - if not args.name: +def _name_of(args, kind: str) -> str: + """The object name a table-scoped type asks with. + + ``_validate_name`` has already refused a run that omitted one: a + ``required`` type whose NAME is missing never reaches this function. The + table is read here as well only so a caller that skipped the validator + still meets the rule rather than sending a ``None`` the introspector would + answer as a wider question than the one asked -- the decision is written + once, in ``_NAME_RULE``, and every reader reads it. + """ + if _NAME_RULE.get(kind) == "required" and not args.name: print( f"Error: Table name is required for '{kind}' introspection", file=sys.stderr, @@ -284,9 +508,13 @@ def _steps(args) -> List[Tuple[str, Dict[str, Any]]]: which type: one type, one question. The table detail that used to be assembled here went with the aggregate type it belonged to. - The NAME each type accepts is decided here too, because it *is* part of - the question: ``views NAME`` is the single-view question, not a filter over - the list. + The NAME each type accepts is *not* decided here. ``_validate_name`` reads + ``_NAME_RULE`` before the connection is opened -- a missing one is an + incomplete invocation, a name the type has no question for is refused + with exit 2 -- so by the time this runs a ``required`` type always has its + name and the branches consume ``args.name`` directly. The branches below + that used to carry their own "Table name is required" sentence now rely on + that, so there is one decision rather than one per branch. """ kind = args.type @@ -324,17 +552,92 @@ def _steps(args) -> List[Tuple[str, Dict[str, Any]]]: if kind == "schemas": return [("list_schemas", {})] + # ---- the nine families shared with the MySQL and PostgreSQL CLIs ----- + # The counters and the listings answer for the whole schema, so none + # takes a name; the graph narrows to one object and answers the whole + # graph without one; the one-object families ask about a table. + if kind == "unused-indexes": + return [("list_unused_indexes", {"schema": None})] + if kind == "table-stats": + return [("list_table_usage_stats", {"schema": None})] + if kind == "runtime": + return [("list_runtime_metrics", {"schema": None})] + if kind == "index-stats": + return [("list_index_usage_stats", {"schema": None})] + if kind == "dependencies": + return [( + "list_object_dependencies", + {"object_name": args.name, "schema": None}, + )] + if kind == "ddl": + return [( + "get_ddl_statement", + {"object_name": _name_of(args, kind), "schema": None}, + )] + if kind == "column-profiles": + return [( + "list_column_data_profiles", + {"table_name": _name_of(args, kind), "schema": None}, + )] + if kind == "value-samples": + if not getattr(args, "column", None): + print( + "Error: --column is required for 'value-samples' introspection", + file=sys.stderr, + ) + sys.exit(1) + return [( + "sample_column_values", + { + "table_name": _name_of(args, kind), + "column_name": args.column, + "schema": None, + **_value_sample_arguments(args), + }, + )] + if kind == "join-coverage": + missing = [ + flag + for attribute, flag, required in _TYPE_OPTIONS["join-coverage"] + if required and not getattr(args, attribute, None) + ] + if missing: + print( + f"Error: {', '.join(missing)} is required for " + "'join-coverage' introspection", + file=sys.stderr, + ) + sys.exit(1) + return [( + "estimate_join_coverage", + { + "left_table": args.left_table, + "left_column": args.left_column, + "right_table": args.right_table, + "right_column": args.right_column, + "schema": None, + }, + )] + # columns / indexes / foreign-keys / extended-properties method = f"list_{kind.replace('-', '_')}" return [( method, - {"table_name": _require_name(args, kind), "schema": None}, + {"table_name": _name_of(args, kind), "schema": None}, )] def _title_for(method: str, kwargs: Dict[str, Any]) -> str: """The heading for one section of output.""" if method in _RESULT_TITLES: + # Two of the shared families name the object in their title: the + # graph may be narrowed to one object, and the data families are + # about one table. The rest describe the database and read the same + # whatever the caller asked. + if method == "list_object_dependencies" and kwargs.get("object_name"): + return f"Dependencies of {kwargs['object_name']}" + if method == "list_column_data_profiles": + return f"Column profiles of {kwargs.get('table_name', '')}" return _RESULT_TITLES[method] if method == "get_view_info": return f"View {kwargs.get('view_name', '')}" @@ -347,6 +650,13 @@ def _title_for(method: str, kwargs: Dict[str, Any]) -> str: return f"Foreign Keys of {name}" if method == "list_extended_properties": return f"Extended Properties of {name}" + if method == "sample_column_values": + column = kwargs.get("column_name", "") + return f"Most frequent values of {name}.{column}" + if method == "estimate_join_coverage": + left = f"{kwargs.get('left_table', '')}.{kwargs.get('left_column', '')}" + right = f"{kwargs.get('right_table', '')}.{kwargs.get('right_column', '')}" + return f"Join coverage of {left} and {right}" return method @@ -415,6 +725,34 @@ async def _require_found_async( _report_missing(missing) +def _require_measured(method: str, kwargs: Dict[str, Any], value: Any) -> None: + """Refuse a ``None`` answer that means "not measured", not zero. + + Two of the shared families answer with a single object or nothing: + ``None`` from the DDL question says the engine will not produce a text, + and ``None`` from the coverage question says the overlap could not be + measured. Both are different answers from an empty row and from a measured + zero, so both leave with 1 rather than printing a row of nulls that a + reader would take for a measurement. + """ + if value is not None: + return + if method == "get_ddl_statement": + print( + f"Error: no DDL available for '{kwargs.get('object_name', '')}'", + file=sys.stderr, + ) + sys.exit(1) + if method == "estimate_join_coverage": + left = f"{kwargs.get('left_table', '')}.{kwargs.get('left_column', '')}" + right = f"{kwargs.get('right_table', '')}.{kwargs.get('right_column', '')}" + print( + f"Error: this server could not measure the overlap of {left} and {right}", + file=sys.stderr, + ) + sys.exit(1) + + def _value_sections( method: str, kwargs: Dict[str, Any], value: Any ) -> List[Tuple[str, Any]]: @@ -482,6 +820,16 @@ def _handle_introspect_sync(args, backend: SQLServerBackend, provider): for method, kwargs in _steps(args): value = getattr(introspector, method)(**kwargs) _require_found(introspector, method, kwargs, value) + _require_measured(method, kwargs, value) + if method == "list_column_data_profiles": + print( + "Note: column profiles are aggregated from the rows themselves " + "(source='aggregate') -- exact, and paid for with a full scan per column.", + file=sys.stderr, + ) + if method == "get_ddl_statement": + _display_ddl(args, provider, value) + continue sections.extend(_value_sections(method, kwargs, value)) _render(provider, sections) @@ -510,6 +858,16 @@ async def _handle_introspect_async(args, backend, provider): for method, kwargs in _steps(args): value = await getattr(introspector, method)(**kwargs) await _require_found_async(introspector, method, kwargs, value) + _require_measured(method, kwargs, value) + if method == "list_column_data_profiles": + print( + "Note: column profiles are aggregated from the rows themselves " + "(source='aggregate') -- exact, and paid for with a full scan per column.", + file=sys.stderr, + ) + if method == "get_ddl_statement": + _display_ddl(args, provider, value) + continue sections.extend(_value_sections(method, kwargs, value)) _render(provider, sections) diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/output.py b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/output.py index 04d0678e..ca71e6c6 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/output.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/output.py @@ -15,6 +15,23 @@ RichOutputProvider = None # type: ignore[misc,assignment] +#: The output formats the introspect, query and status subcommands offer. +#: +#: The provider factory below is what makes them available: a format listed +#: here must map to a provider, and a provider nobody is offered is unreachable. +#: Defining the list once, beside the factory that has to be able to produce +#: every member, is what keeps "offered" and "producible" from drifting apart. +OUTPUT_CHOICES = ["table", "json", "csv", "tsv"] + +#: The narrower set ``info`` offers. +#: +#: ``info`` prints one object's facts, a shape the plain-text provider lays out +#: better than a record table; offering csv and tsv there would advertise two +#: renderings of a single row. The name says which surface it belongs to, so +#: the two lists are not one name answering two questions. +INFO_OUTPUT_CHOICES = ["table", "json"] + + def create_provider(output_format: str, ascii_borders: bool = False): """Factory function to get the correct output provider. diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/query.py b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/query.py index afba3016..b7579537 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/query.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/query.py @@ -12,9 +12,7 @@ from rhosocial.activerecord.backend.impl.sqlserver.backend import SQLServerBackend from rhosocial.activerecord.backend.errors import ConnectionError, QueryError from .connection import add_connection_args, resolve_connection_config_from_args -from .output import create_provider, RICH_AVAILABLE - -OUTPUT_CHOICES = ['table', 'json', 'csv', 'tsv'] +from .output import OUTPUT_CHOICES, create_provider, RICH_AVAILABLE def create_parser(subparsers): diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/status.py b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/status.py index 819553cc..71207c76 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/cli/status.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/cli/status.py @@ -12,9 +12,7 @@ from rhosocial.activerecord.backend.impl.sqlserver.backend import SQLServerBackend from rhosocial.activerecord.backend.errors import ConnectionError, QueryError from .connection import add_connection_args, resolve_connection_config_from_args -from .output import create_provider, RICH_AVAILABLE - -OUTPUT_CHOICES = ['table', 'json', 'csv', 'tsv'] +from .output import OUTPUT_CHOICES, create_provider, RICH_AVAILABLE STATUS_TYPES = ["all", "config", "performance", "connections", "storage", "databases", "users"] @@ -126,13 +124,13 @@ def _format_size(size_bytes: int) -> str: def _handle_status_sync(args, backend: SQLServerBackend, provider): """Handle status subcommand synchronously.""" from rhosocial.activerecord.backend.introspection.status import StatusCategory - from ..introspection.status_introspector import SyncSQLServerStatusIntrospector + from ..introspection.status_introspector import SQLServerStatusIntrospector try: backend.connect() backend.introspect_and_adapt() - status_introspector = SyncSQLServerStatusIntrospector(backend) + status_introspector = SQLServerStatusIntrospector(backend) status_type = args.type if status_type == "all": diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/__init__.py b/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/__init__.py index 9640bdd9..79b1ce35 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/__init__.py @@ -1,9 +1,9 @@ # src/rhosocial/activerecord/backend/impl/sqlserver/introspection/__init__.py """SQL Server schema introspection.""" -from .introspector import SyncSQLServerIntrospector, AsyncSQLServerIntrospector +from .introspector import SQLServerIntrospector, AsyncSQLServerIntrospector from .status_introspector import ( - SyncSQLServerStatusIntrospector, + SQLServerStatusIntrospector, AsyncSQLServerStatusIntrospector, ) from .types import ( @@ -14,9 +14,9 @@ ) __all__ = [ - "SyncSQLServerIntrospector", + "SQLServerIntrospector", "AsyncSQLServerIntrospector", - "SyncSQLServerStatusIntrospector", + "SQLServerStatusIntrospector", "AsyncSQLServerStatusIntrospector", "SQLServerExtendedPropertyInfo", "SQLServerRoutineInfo", diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/introspector.py index 9f333c2e..07a607ee 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/introspector.py @@ -1,7 +1,7 @@ # src/rhosocial/activerecord/backend/impl/sqlserver/introspection/introspector.py """SQL Server concrete introspectors. -Implements SyncAbstractIntrospector and AsyncAbstractIntrospector for SQL Server +Implements AbstractIntrospector and AsyncAbstractIntrospector for SQL Server using INFORMATION_SCHEMA and sys.* catalog views. """ @@ -11,12 +11,12 @@ from rhosocial.activerecord.backend.introspection.base import ( IntrospectorMixin, - SyncAbstractIntrospector, + AbstractIntrospector, AsyncAbstractIntrospector, ) from rhosocial.activerecord.backend.introspection.cache import IntrospectionCache from rhosocial.activerecord.backend.introspection.executor import ( - SyncIntrospectorExecutor, + IntrospectorExecutor, AsyncIntrospectorExecutor, ) from rhosocial.activerecord.backend.introspection.types import ( @@ -287,6 +287,15 @@ def parse_tables( ``INFORMATION_SCHEMA``-shaped question still parses. An unknown code reads as a base table, which is the least surprising answer for a relation this backend listed as one. + + The three storage slots arrive as the subqueries + ``_partition_stats_bytes`` projected. A subquery that found no + partition rows -- a view has none -- answers ``NULL``, which the + driver returns as ``None``, so "storage this engine does not track + for this object" stays ``None`` rather than being written as zero. + ``row_count`` is deliberately not answered: an exact count is a scan, + and an estimate is a table the engine keeps no estimate in, so the + slot says so instead of guessing. """ target = schema or self.default_schema return [ @@ -298,6 +307,9 @@ def parse_tables( TableType.BASE_TABLE, ), comment=row.get("table_comment"), + size_bytes=row.get("size_bytes"), + data_size_bytes=row.get("data_size_bytes"), + index_size_bytes=row.get("index_size_bytes"), ) for row in rows ] @@ -615,15 +627,15 @@ def _as_decimal(value: Any) -> Optional[Decimal]: return None -class SyncSQLServerIntrospector( - SQLServerIntrospectorMixin, SyncAbstractIntrospector +class SQLServerIntrospector( + SQLServerIntrospectorMixin, AbstractIntrospector ): """Synchronous introspector for SQL Server backends.""" def __init__( self, backend: Any, - executor: SyncIntrospectorExecutor, + executor: IntrospectorExecutor, cache: Optional[IntrospectionCache] = None, ) -> None: super().__init__(backend, executor, cache) diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/status_introspector.py b/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/status_introspector.py index 26d11446..9e117dd3 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/status_introspector.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/status_introspector.py @@ -23,7 +23,7 @@ ConnectionInfo, StorageInfo, SessionInfo, - SyncAbstractStatusIntrospector, + AbstractStatusIntrospector, AsyncAbstractStatusIntrospector, ) @@ -250,7 +250,7 @@ def _session_info_sql(self) -> Tuple[str, tuple]: -class SyncSQLServerStatusIntrospector(SQLServerStatusIntrospectorMixin, SyncAbstractStatusIntrospector): +class SQLServerStatusIntrospector(SQLServerStatusIntrospectorMixin, AbstractStatusIntrospector): """Synchronous SQL Server status introspector.""" def _execute_query(self, sql: str) -> List[tuple]: diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/types.py b/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/types.py index 568817c8..2ca33e15 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/types.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/introspection/types.py @@ -21,7 +21,10 @@ from dataclasses import dataclass from datetime import datetime from decimal import Decimal -from typing import Optional +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from rhosocial.activerecord.backend.introspection import SequenceInfo @dataclass @@ -102,6 +105,42 @@ class SQLServerSequenceInfo: is_cached: bool = False cache_size: Optional[int] = None + def to_core(self) -> "SequenceInfo": + """The cross-dialect minimum this answer carries. + + Core's ``SequenceInfo`` names the smallest shape every engine with + sequences answers, and this is the projection onto it: the fields + with a one-to-one reading are copied, ``current_value`` lands on + ``last_value`` and ``is_cycling`` on ``cycle``. A catalog value that + is not integral stays ``None`` here and complete on this class -- + SQL Server permits a sequence declared on ``decimal(10,2)``, whose + increment cannot be an integer, and the common minimum does not + widen to hold one engine's non-integral sequence. ``is_cached`` and + the exact numeric type stay on this side too: caching is a storage + detail the minimum has no field for + (``.claude/rules/introspection.md`` section 5). + """ + from rhosocial.activerecord.backend.introspection import SequenceInfo + + def as_int(value: Optional[Decimal]) -> Optional[int]: + if value is None: + return None + integral = value.to_integral_value() + return int(integral) if integral == value else None + + return SequenceInfo( + name=self.name, + schema=self.schema, + data_type=self.data_type, + start_value=as_int(self.start_value), + minimum_value=as_int(self.minimum_value), + maximum_value=as_int(self.maximum_value), + increment=as_int(self.increment), + cycle=self.is_cycling, + cache_size=self.cache_size, + last_value=as_int(self.current_value), + ) + __all__ = [ "SQLServerExtendedPropertyInfo", diff --git a/src/rhosocial/activerecord/backend/impl/sqlserver/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/sqlserver/mixins/introspection.py index ae357b5f..bc5af7cc 100644 --- a/src/rhosocial/activerecord/backend/impl/sqlserver/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/sqlserver/mixins/introspection.py @@ -33,20 +33,29 @@ from typing import List, Optional, Tuple, TYPE_CHECKING +from rhosocial.activerecord.backend.introspection.types import ProfileSource + from ..query_nodes import SQLServerNodeBuilderMixin if TYPE_CHECKING: # pragma: no cover from rhosocial.activerecord.backend.dialect import SQLDialectBase from rhosocial.activerecord.backend.introspection.types import IntrospectionScope from rhosocial.activerecord.backend.expression.introspection import ( + ColumnDataProfileQueryStatement, ColumnInfoQueryStatement, + ColumnValueSampleQueryStatement, DatabaseInfoQueryStatement, ForeignKeyQueryStatement, IndexInfoQueryStatement, + JoinCoverageQueryStatement, TableListQueryStatement, TriggerListQueryStatement, ViewInfoQueryStatement, ViewListQueryStatement, + UnusedIndexesQueryStatement, + IndexUsageStatsQueryStatement, + RuntimeMetricsQueryStatement, + ObjectDependencyQueryStatement, ) from ..expression.introspection import ( SQLServerExtendedPropertyListQueryStatement, @@ -175,6 +184,20 @@ def supports_extended_property_introspection(self) -> bool: """ return True + def supports_table_storage_introspection(self) -> bool: + """``sys.dm_db_partition_stats`` answers per-table storage. + + SQL Server's storage is row-plus-index shaped -- a heap or clustered + index holds the rows, nonclustered indexes hold the rest -- so all + four narrow slots of ``IntrospectionTableInfo`` can be answered from + one partition-stats read per table: ``size_bytes`` the total, + ``data_size_bytes`` the table's own partitions and + ``index_size_bytes`` the nonclustered ones' pages. Before this was + declared, the listing left every slot ``None`` while the engine was + able to answer; a caller gating on the switch now learns that. + """ + return True + def get_supported_introspection_scopes(self) -> List["IntrospectionScope"]: """The catalogue families this dialect answers, as core names them. @@ -222,6 +245,47 @@ def _cast_server_property(self, name: str, alias: Optional[str] = None): ).cast(SQLServerNVarCharType(self._node_dialect, 128)) return node.as_(alias) if alias else node + def _partition_stats_bytes(self, alias: str, *, heap: bool = False, nonclustered: bool = False): + """One storage slot, read from ``sys.dm_db_partition_stats``. + + The view counts one row per partition of one index, which is where + SQL Server answers "how much space does this table occupy": the + partitions of ``index_id`` 0 (a heap) or 1 (a clustered index) are + the table's own storage, and everything above 1 is a nonclustered + index's. Which bucket *alias* asks for is the two flags; with + neither, every partition of the object is summed -- the total. + + A view has no partition rows at all, so the scalar subquery answers + NULL and the slot stays ``None``: an object whose storage is not a + question the engine answers for it is a different answer from zero + bytes, and claiming the zero would describe storage it does not have. + + The page size is a constant of this engine -- SQL Server pages are + 8 KiB with no configuration switch, unlike PostgreSQL's or Oracle's + -- so the multiplication is a fact of the engine rather than a read + that could come back unknown. + """ + bucket = None + if heap: + bucket = self._in(self._col("index_id", "ps"), (0, 1)) + elif nonclustered: + bucket = self._gt( + self._col("index_id", "ps"), self._lit(1, inline=True) + ) + conditions = [ + self._eq(self._col("object_id", "ps"), self._col("object_id", "o")), + ] + if bucket is not None: + conditions.append(bucket) + return self._mul( + self._subquery( + [self._fn("SUM", self._col("used_page_count", "ps"))], + from_=self._relation("dm_db_partition_stats", "sys", alias="ps"), + where=self._and(*conditions), + ), + self._lit(8192, inline=True), + ).as_(alias) + def format_database_info_query_statement( self, expr: "DatabaseInfoQueryStatement" ) -> Tuple[str, tuple]: @@ -273,6 +337,14 @@ def format_table_list_query_statement( join is a ``LEFT JOIN`` on the object-level property alone (``minor_id = 0``), so a table without a comment still row-appears and a column-level description cannot be mistaken for the table's. + + The storage slots ride the same listing as three scalar subqueries + rather than a join: ``sys.dm_db_partition_stats`` holds one row per + partition per index, and a GROUP BY over it would fold the object + kinds -- a view, which has no partitions, would vanish from the + listing instead of answering ``None`` for its storage. A subquery per + slot keeps every row and says plainly which slots a view cannot + answer; see :meth:`_partition_stats_bytes`. """ params = expr.get_params() target = params.get("schema") or DEFAULT_SCHEMA @@ -325,6 +397,9 @@ def format_table_list_query_statement( self._cast(self._col("value", "ep"), "NVARCHAR(4000)").as_( "table_comment" ), + self._partition_stats_bytes("size_bytes"), + self._partition_stats_bytes("data_size_bytes", heap=True), + self._partition_stats_bytes("index_size_bytes", nonclustered=True), ], from_=joined, where=self._and(*conditions), @@ -1004,3 +1079,809 @@ def format_sequence_list_query_statement( where=self._eq(self._col("name", "s"), self._lit(target)), order_by=[self._col("name", "q")], ) + + # ========== Operational families ========== + + def supports_unused_indexes_detection(self) -> bool: + """SQL Server answers from ``sys.dm_db_index_usage_stats``, 2005 on. + + The dynamic management views are a live-instance view rather than a + schema-scoped catalogue, which is the dependency this round recorded + as pending. It is carried: the join through ``sys.objects`` and + ``sys.schemas`` applies the schema filter the statement needs, the + way the per-table storage read already reaches + ``sys.dm_db_partition_stats`` -- one construct, the same shape as the + catalogue questions this module already asks. + + The counters are *since the view was last flushed*, which is the + engine's own window and not a resettable one: ``DBCC + SQLPERF('sys.dm_os_wait_stats',CLEAR)`` clears some counters and the + index usage view flushes on its own schedule after a restart. An index + absent from the answer is absence of evidence, never proof of use, and + the DMV only reports the indexes it knows about. + + Reading the DMV needs ``VIEW DATABASE STATE`` (or ``VIEW SERVER + STATE``); without it the view returns only the objects the caller + owns. A caller with less privilege gets a shorter list, not an error, + which is why the privilege is stated here rather than turned into a + probe: the probe can ask a version, not a grant, and a short list is + an answer about what the caller may see. + """ + return self.version_at_least((9, 0, 0)) + + def supports_index_stats(self) -> bool: + """SQL Server answers per-index counters from the same DMV, 2005 on. + + ``user_seeks``/``user_scans``/``user_lookups`` are the three read + shapes and their sum is what ``scans`` answers; ``user_updates`` rides + the row into ``extra``. There is no per-index "rows read" counter in + this view, so ``rows_read`` and ``rows_fetched`` stay ``None`` rather + than being filled with a seek or a scan count -- an operation is not a + row and the two must not be merged into one slot. + + The privilege and the window are the ones + :meth:`supports_unused_indexes_detection` states, because it is the + same view. + """ + return self.version_at_least((9, 0, 0)) + + def supports_table_stats(self) -> bool: + """SQL Server keeps no per-table usage counter, so the probe refuses. + + The inquiry covered every release in the matrix (2017, 2019 and + 2022). The engine's counters are per *index*: + ``sys.dm_db_index_usage_stats`` rows are keyed by object and index, + and summing them per table would be an aggregate wearing the shape of + a measured counter -- a reader asking "how many scans did this table + take" would receive a number no engine ever counted, and the family's + promise is that each figure is the engine's own. + + Two of the slots the family names are not tracked at all. Sequential + scans are not distinguishable from indexed ones at the table level, + and inserts and deletes are recorded nowhere -- ``user_updates`` is + the only write shape the DMV carries, and it counts writes of every + kind. So the honest answer is that the engine does not measure this, + and the probe says so rather than projecting a derived figure. + + The row estimate is a different source again: + ``sys.dm_db_partition_stats`` keeps an in-memory count whose refresh + is the engine's own, so it stays where it already is -- on the table + listing's ``row_count`` slot -- rather than being copied beside + counters it has nothing to do with. + """ + return False + + def supports_runtime_stats(self) -> bool: + """SQL Server answers runtime measurements from its performance counters. + + ``sys.dm_os_performance_counters`` names what is actually being + counted on this instance -- one row per counter, grouped by the + object that owns it -- which makes it the honest source for a "what is + alive" listing: a counter not being maintained produces no row, and + therefore no number, rather than a zero that would read as "idle". + ``cntr_type`` carries the engine's own classification and rides the + row as ``kind``; the counter's owning object is projected as + ``source``, because these figures are since the view was last flushed + and the window is the engine's to name. + + Reading it needs ``VIEW SERVER STATE``; without that grant the view + answers nothing, and a listing that silently loses every row there is + a privilege statement rather than a measurement. The probe cannot ask + a grant -- only a version -- so the privilege is stated here and the + version floor is the one the view needs. + """ + return self.version_at_least((9, 0, 0)) + + def supports_object_dependencies(self) -> bool: + """SQL Server answers the object dependency graph, 2008 on. + + ``sys.sql_expression_dependencies`` records, per reference, which + object depends on which *named* entity -- a view on a table, a + procedure on a table, a function used by a trigger. It is a real graph + and a partial one: by-name references resolve only to objects the + caller can see, and the view is scoped to the database, so a + cross-database reference names an entity rather than an object. + + It does not record every other kind of coupling -- a foreign key is + not a row here, and an object bound through ``sp_depends``-style + dynamic SQL is invisible to it. A caller asking "what breaks if I drop + this" gets the referential half plus the resolved-name half, and the + docstring says which half. + """ + return self.version_at_least((10, 0, 0)) + + def supports_ddl_extraction(self) -> bool: + """SQL Server will not hand back the DDL that recreates an object. + + The engine has no per-object DDL retrieval call of the kind + PostgreSQL's ``DBMS_METADATA`` and MySQL's ``SHOW CREATE TABLE`` are. + ``OBJECT_DEFINITION`` and ``sys.sql_modules`` hold the *body* of a + routine, a view or a trigger -- text the engine stores because it + executes it -- and a table's columns, constraints, indexes and + storage are recorded as rows across ``sys.columns``, + ``sys.indexes``, ``sys.default_constraints`` and more. Reassembling a + ``CREATE TABLE`` from them is a reconstruction, and this family's rule + is that a reconstruction is not an answer: ``DDLStatement.native`` + exists to say which of the two a caller is looking at, and an + assembly that used it would be claiming the engine said something it + did not. + + So the probe answers ``False`` and the CLI refuses the question at the + capability gate rather than rendering a partial text. The four + operational families this round *does* answer are listed above; this + one stays a declared gap, and the roundtrip matrix in this repository + keeps its pin for ``DDLStatementQueryStatement`` with this reason. + """ + return False + + def _usage_stats_source(self, with_indexes: bool): + """The per-index DMV joined to the catalogue that names its objects. + + ``sys.dm_db_index_usage_stats`` addresses everything by id -- + ``database_id``, ``object_id``, ``index_id`` -- and carries no name + and no schema. The join through ``sys.objects`` and ``sys.schemas`` + is what turns those ids into the address a caller reads, and it is + also where the schema filter is applied: the DMV covers every + database on the instance, so a caller asking about one schema must + not be handed every schema's indexes. + + ``sys.indexes`` is joined as well when the caller reads the index's + *name*: the DMV has no ``index_id``-to-name mapping of its own, and + the heap's ``index_id`` 0 has no row there at all. + + The database is *not* filtered here, and that is deliberate. The DMV + keeps one row per index of the connected database, and a + ``database_id`` filter would have to name it -- either through + ``DB_ID()``, whose value the statement would then depend on, or + through a second read. The join to ``sys.schemas`` already narrows to + one schema of this database, which is the narrowing the statement was + asked for. + """ + usage = self._relation("dm_db_index_usage_stats", "sys", alias="u") + joined = self._join( + self._join( + usage, + self._relation("objects", "sys", alias="o"), + condition=self._eq( + self._col("object_id", "u"), self._col("object_id", "o") + ), + ), + self._relation("schemas", "sys", alias="s"), + condition=self._eq( + self._col("schema_id", "o"), self._col("schema_id", "s") + ), + ) + if with_indexes: + joined = self._join( + joined, + self._relation("indexes", "sys", alias="i"), + condition=self._and( + self._eq(self._col("object_id", "u"), self._col("object_id", "i")), + self._eq(self._col("index_id", "u"), self._col("index_id", "i")), + ), + ) + return joined + + def _read_counter_sum(self): + """The three read shapes added: seeks, scans and lookups. + + One ``scans`` figure, because the engine's own counters are three + columns of the same event and the family names one slot. Adding them + is not a derived measurement; it is the same counter, spelled in three + parts, and the docstring of the family that reads it says which parts. + """ + return self._add( + self._add(self._col("user_seeks", "u"), self._col("user_scans", "u")), + self._col("user_lookups", "u"), + ) + + def format_unused_indexes_query_statement( + self, expr: "UnusedIndexesQueryStatement" + ) -> Tuple[str, tuple]: + """Format the unused-indexes listing from the index usage DMV. + + The predicates are the answer to "unused" as this engine means it: + ``index_id > 0`` excludes a heap's row (``index_id`` 0 is not an index + and never appears in ``sys.indexes``), and the sum of the three read + counters being zero says no query the engine counted has used it. + + The name of the index is read through ``sys.indexes`` rather than + reconstructed from the DMV's ids, and the object and schema through + the join above. See :meth:`_usage_stats_source`. + """ + params = expr.get_params() + target = params.get("schema") or DEFAULT_SCHEMA + + return self._select( + [ + self._col("name", "s", alias="schema_name"), + self._col("name", "o", alias="table_name"), + self._col("name", "i", alias="index_name"), + ], + from_=self._usage_stats_source(with_indexes=True), + where=self._and( + self._eq(self._col("name", "s"), self._lit(target)), + self._gt(self._col("index_id", "u"), self._lit(0, inline=True)), + self._eq(self._read_counter_sum(), self._lit(0, inline=True)), + ), + order_by=[ + self._col("name", "s"), + self._col("name", "o"), + self._col("name", "i"), + ], + ) + + def format_index_usage_stats_query_statement( + self, expr: "IndexUsageStatsQueryStatement" + ) -> Tuple[str, tuple]: + """Format per-index counters from the same DMV. + + ``scans`` is the shared slot the three read columns sum into, and + ``user_updates`` rides the row into ``extra`` under its own name -- + an update is not a scan, and the two are not interchangeable. The + engine keeps no per-index row counter, so ``rows_read`` and + ``rows_fetched`` are simply not projected and stay ``None``; see + :meth:`supports_index_stats`. + """ + params = expr.get_params() + target = params.get("schema") or DEFAULT_SCHEMA + + return self._select( + [ + self._col("name", "s", alias="schema_name"), + self._col("name", "o", alias="table_name"), + self._col("name", "i", alias="index_name"), + self._read_counter_sum().as_("scans"), + self._col("user_updates", "u", alias="writes"), + ], + from_=self._usage_stats_source(with_indexes=True), + where=self._and( + self._eq(self._col("name", "s"), self._lit(target)), + self._gt(self._col("index_id", "u"), self._lit(0, inline=True)), + ), + order_by=[ + self._col("name", "s"), + self._col("name", "o"), + self._col("name", "i"), + ], + ) + + def format_runtime_metrics_query_statement( + self, expr: "RuntimeMetricsQueryStatement" + ) -> Tuple[str, tuple]: + """Format the runtime listing from the instance's performance counters. + + No schema filter: the counters describe a running instance and carry + no schema column, so a filter here would refuse a correct query for a + nonsensical reason -- the same reasoning the storage slots use when + they leave a view's storage unmeasured. + + ``cntr_value`` is the number and ``cntr_type`` the engine's + classification of it, projected as ``kind`` so a caller can group + without learning the DMV's vocabulary. ``object_name`` is the counter + group that owns it (``SQLServer:Buffer Manager`` and friends) and is + projected as ``source``; ``instance_name`` -- the per-node or per-lru + split that makes two rows of the same counter distinguishable -- is + projected under its own name and travels in ``extra``. + """ + return self._select( + [ + self._col("counter_name", alias="name"), + self._col("cntr_value", alias="value"), + self._col("cntr_type", alias="kind"), + self._col("object_name", alias="source"), + self._col("instance_name"), + ], + from_=self._relation("dm_os_performance_counters", "sys"), + order_by=[self._col("object_name"), self._col("counter_name")], + ) + + def format_object_dependency_query_statement( + self, expr: "ObjectDependencyQueryStatement" + ) -> Tuple[str, tuple]: + """Format the dependency graph from by-name entity references. + + ``referencing_id`` is the object that depends and + ``referenced_entity_name`` is the entity it names, so the two sides + come from different columns of the same row -- which is why the join + is on the referencing object rather than on the referenced one: the + referenced thing is a *name*, not an id, and a second join would have + to resolve it. + + ``referenced_class_desc`` says what kind of thing was named (OBJECT, + COLUMN, TYPE) and rides the row as ``kind``. A ``referenced_server_name`` + or ``referenced_database_name`` other than this one means the referenc + leaves the database; those rows are kept and their names travel in + ``extra``, because dropping an object with a cross-database + dependant is exactly the case the graph exists to show. + + """ + params = expr.get_params() + target = params.get("schema") or DEFAULT_SCHEMA + object_name = params.get("object_name") + + joined = self._join( + self._relation("sql_expression_dependencies", "sys", alias="d"), + self._relation("objects", "sys", alias="o"), + condition=self._eq( + self._col("referencing_id", "d"), self._col("object_id", "o") + ), + ) + joined = self._join( + joined, + self._relation("schemas", "sys", alias="s"), + condition=self._eq( + self._col("schema_id", "o"), self._col("schema_id", "s") + ), + ) + + conditions = [self._eq(self._col("name", "s"), self._lit(target))] + if object_name: + conditions.append( + self._eq(self._col("name", "o"), self._lit(object_name)) + ) + + return self._select( + [ + self._col("name", "s", alias="schema_name"), + self._col("name", "o", alias="dependent"), + self._col("referenced_entity_name", "d", alias="referenced"), + self._col("type_desc", "o", alias="dependent_type"), + self._col("referenced_class_desc", "d", alias="kind"), + ], + from_=joined, + where=self._and(*conditions), + order_by=[ + self._col("name", "s"), + self._col("name", "o"), + self._col("referenced_entity_name", "d"), + ], + ) + + # ========== Data Profiles (row-shaped answers) ========== + + def supports_column_data_profile(self) -> bool: + """SQL Server answers a column data profile by aggregating the rows. + + SQL Server publishes statistics, and they are the reason this probe is + worth a paragraph rather than a sentence: ``sys.stats`` names a + statistics object, ``sys.stats_columns`` maps that object to one or + more columns, and ``sys.dm_db_stats_properties`` reports what the last + update of *that object* measured -- ``rows``, ``rows_sampled``, + ``steps``, ``modification_counter``, ``last_updated`` and + ``persisted_sample_percent``. The key is the object, not the column. A + single-column statistics object is a fair proxy for its column, but a + two-column one describes the pair's leading prefix, and the same column + can appear in several objects with different samples, so summing or + picking one would invent a per-column fact the catalogue does not hold. + + What is missing is sharper than what is keyed oddly. There is no + per-column null count anywhere in the catalogue -- nullability is a + *declaration* in ``sys.columns``, never a measurement -- and no + per-column distinct count either; ``sys.dm_db_stats_histogram`` + (2016+) reports ``range_high_key``/``equal_rows``/``distinct_range_rows`` + per step of one object, which is a bucket summary for that object's + leading column, not a count of the values in a column. The one figure + the catalogue could contribute, ``last_updated``, would be the vintage + of a statistics object, not of the profile -- and this family's rule is + that a figure travels with the source and vintage that produced it. + + So all five figures come from one place, and it is the rows: exact, + and paid for in time. The label is ``aggregate``, projected into each + row rather than stamped onto the result, so nothing here can be + mistaken for an estimate of somebody's last sample. + """ + return True + + def supports_column_value_sample(self) -> bool: + """SQL Server answers a value listing by aggregating with a limit. + + The nearest catalogue answer is ``sys.dm_db_stats_histogram``, and it + answers a different question: a histogram step's ``equal_rows`` counts + the rows equal to that step's ``range_high_key``, so the list it could + produce would be bucket boundaries with estimated weights -- only for + the leading column of a statistics object, and only for the values + whose boundaries survived bucketing. A ``GROUP BY`` answers what the + caller asked for, exactly, at the cost of a grouped scan. + + The cap rides the dialect's pagination clause, ``OFFSET 0 ROWS FETCH + NEXT n ROWS ONLY``, which the engine parses only from 2012 on. That + floor is asked of the version accessor rather than left to a runtime + syntax error: a server below it is refused at the capability question + with "cannot answer", which is a different and truer answer than a + query that fails to parse. Every release this repository tests + against (2017, 2019, 2022 and 2025) is well past it, so the gate + changes nothing there and keeps an older server from an execution + error. + """ + return self.version_at_least((11, 0, 0)) + + def supports_join_coverage(self) -> bool: + """SQL Server answers overlap by counting distinct values on both sides. + + Exact: two ``COUNT(DISTINCT ...)`` denominators and one intersection, + with ``NULL`` excluded on both sides because a NULL is not a value that + could ever join. Answering ``True`` is right because the probe asks + whether the engine *can* answer, not whether it can answer cheaply -- + the cost, two scans of each column, belongs in the docstring. + """ + return True + + def _user_relation(self, table_name: str, target: str, alias: Optional[str] = None): + """A user table as a FROM source, named in *target*'s schema. + + The opposite of the ``sys`` relations above: this one is whatever its + owner declared, and it lives in a caller-named schema, so both parts go + through the dialect's identifier renderer. ``target`` is never empty -- + the formatters below fall back to :data:`DEFAULT_SCHEMA` the same way + every other statement in this module does. + + ``alias`` is what tells the two sides of a coverage measurement apart; + it is quoted like every other identifier. + """ + return self._relation(table_name, target, alias=alias) + + def _profile_cast(self, node, type_name: str, alias: Optional[str] = None): + """``CAST(node AS type_name)`` for a node that is not a column. + + :meth:`_cast` asks the node to cast *itself*, which only a reference + carrying a declared type can do; a literal, a ``CASE`` and a function + call are values, so the cast is built around them instead. The type is + parsed by the dialect's own parser, so the spelling in the statement is + the one this backend uses everywhere else. + """ + from rhosocial.activerecord.backend.expression.core import CastExpression + from rhosocial.activerecord.backend.expression.types._base import DataType + + dialect = self._node_dialect + return CastExpression( + dialect, node, DataType.parse_data_type_str(dialect, type_name), alias=alias + ) + + def _data_profile_label(self, text: Optional[str], alias: str): + """A constant label as a named select-list entry. + + ``Literal`` is deliberately unaliasable -- a value has nothing to name + (see :class:`~...expression.core.Literal`) -- so the constant is cast + to text, which is the smallest node in the tree that produces a named + result. The cast is not there for its type: it is there so the label + can be called ``source``, which is what the parser reads. The width is + stated because SQL Server's ``CHAR`` and ``NVARCHAR`` default to length + 1 in a ``CAST``, which would truncate the word to its first character. + """ + return self._profile_cast( + self._lit(text, inline=True), "NVARCHAR(128)", alias + ) + + def _profile_arm(self, table_name: str, target: str, column_name: str): + """One column's arm of the profile, as a query node. + + The seven entries are the family's contract: the column's own name, the + null count, the distinct count, the two endpoints, the row count the + parser divides by, and the provenance. + + Three type decisions are written into this projection rather than left + to the reader. The null flag is cast to ``BIGINT`` before it is summed, + and the row count is ``COUNT_BIG(*)`` rather than ``COUNT(*)``, because + SQL Server's ``SUM`` and ``COUNT`` return ``INT``: a table with more + than 2,147,483,647 rows -- exactly the table worth profiling -- would + overflow the very counts the profile exists to report. The two + endpoints are cast to ``NVARCHAR(MAX)`` for a reason that belongs to + the union rather than to this arm, and is stated at + :meth:`format_column_data_profile_query_statement`. + """ + from rhosocial.activerecord.backend.expression import ( + CaseExpression, + CountFunction, + ) + + column = self._col(column_name) + null_flag = CaseExpression( + self._node_dialect, + cases=[(column.is_null(), self._lit(1, inline=True))], + else_result=self._lit(0, inline=True), + ) + return self._query( + [ + # The name is bound rather than written into the statement: it + # is a value, and a non-prefixed literal would be typed with + # the database's code page, so an identifier outside that page + # would arrive as question marks. A parameter is typed Unicode + # by the driver, and it keeps the caller's text out of the SQL. + self._profile_cast( + self._lit(column_name), "NVARCHAR(128)", "column_name" + ), + self._fn( + "SUM", self._profile_cast(null_flag, "BIGINT") + ).as_("null_count"), + CountFunction(self._node_dialect, column, distinct=True).as_( + "distinct_count" + ), + self._profile_cast( + self._fn("MIN", column), "NVARCHAR(MAX)", "min_value" + ), + self._profile_cast( + self._fn("MAX", column), "NVARCHAR(MAX)", "max_value" + ), + self._fn("COUNT_BIG", self._row_count_wildcard()).as_("row_count"), + self._data_profile_label(ProfileSource.AGGREGATE.value, "source"), + ], + from_=self._user_relation(table_name, target), + ) + + def format_column_data_profile_query_statement( + self, expr: "ColumnDataProfileQueryStatement" + ) -> Tuple[str, tuple]: + """Format a per-column profile by aggregating the rows themselves. + + The catalogue was audited and refused -- see + :meth:`supports_column_data_profile` for what is keyed by statistics + object and what is simply absent -- so every figure is counted and the + row says so. The ``aggregate`` label is projected *into* each row + rather than stamped onto the result afterwards: a consumer comparing a + SQL Server profile with a PostgreSQL estimate can tell the two apart + without knowing which engine it asked. + + One arm per column, over the column list the expression carries: SQL + cannot pivot ``SUM(CASE WHEN col IS NULL ...)`` across columns it was + never told about, so an aggregating dialect genuinely needs that list + where a catalogue-based one does not. + + The one place in this module where a set operator is written as text. + Every arm is a real :class:`~...expression.statements.dql.QueryStatement` + -- the renderer owns its projection, its identifiers and its relation + -- because a ``UNION ALL`` over a column list known only at run time + has no node to represent it: the column list is a tuple on the + expression, not an operand in the tree. So each arm is rendered on its + own, its parameters are concatenated in arm order, and the rendered + arms are joined by the set operator's word. Nothing else here is text. + + Why the endpoints are cast to text, which is the one place this differs + from the MySQL-family implementation. A union resolves *one* result type + per column position by data-type precedence and implicitly converts the + other arms to it, so the ``min_value`` slot would have to hold an + ``int`` from one arm and a ``varchar`` from another: ``int`` wins, and + the text value is converted to a number, which fails. A ``decimal`` + beside a ``datetime`` fails the same way in the other direction. + Nothing in the formatter knows a column's declared type, so there is no + per-type cast to choose; text is the one target every scalar column + converts to, and the endpoints arrive as the engine's default rendering + of the value. The figures are exact; the Python types are the union's, + and a ``datetime`` endpoint renders without its sub-minute part because + that is what SQL Server's style-free conversion produces. + + ``null_count`` rather than a fraction: dividing by a row count that may + legitimately be zero belongs in the parser, not in a SQL expression + that would then have to guard against it. ``SUM`` over no rows is NULL, + so an empty table answers "not measured" for the fraction rather than + ``0.0``, which would claim a measurement nobody made. + + A table holding an ``xml``, ``text``/``ntext``/``image`` or CLR-typed + column cannot be profiled this way at all: SQL Server refuses ``MIN`` + and ``MAX`` on those types, and the statement fails rather than + answering a partial profile. That is the honest limit of an aggregate, + and it is stated here rather than discovered as a server error. + + An empty column list answers with no rows. That case cannot arise from + a real table -- SQL Server requires at least one column -- but the + branch has to render, so it is a statement that selects no rows rather + than a special-case string. + """ + params = expr.get_params() + target = params.get("schema") or DEFAULT_SCHEMA + + arms = [] + parameters: "list" = [] + for name in expr.columns: + sql, arm_parameters = self._profile_arm( + params.get("table_name"), target, name + ).to_sql() + arms.append(sql) + parameters.extend(arm_parameters) + + if not arms: + return self._query( + [self._data_profile_label(None, "column_name")], + where=self._eq( + self._lit(0, inline=True), self._lit(1, inline=True) + ), + ).to_sql() + + return " UNION ALL ".join(arms), tuple(parameters) + + def format_column_value_sample_query_statement( + self, expr: "ColumnValueSampleQueryStatement" + ) -> Tuple[str, tuple]: + """Format a most-frequent-values listing with a grouped scan. + + Nothing usable is precomputed -- see + :meth:`supports_column_value_sample` for what a histogram step does and + does not answer -- so this costs a ``GROUP BY`` over the column plus a + sort of the groups: the same order as the profile itself, which is why + the two are separate probes rather than one call that would make the + cheap half pay for the dear one. + + ``ORDER BY COUNT_BIG(*) DESC`` is what makes the answer "the most + frequent values" rather than "the first values". ``COUNT_BIG`` rather + than ``COUNT`` for the same reason as the profile: the count of a + value's rows can pass 2,147,483,647 on a large table, and an overflow + there would be reported as an error rather than read as a large group. + Ties break by the value so two runs of the same query agree. + + The cap rides the dialect's pagination clause, ``OFFSET 0 ROWS FETCH + NEXT n ROWS ONLY``, which is SQL Server 2012+ syntax; ``n`` is an + ``int`` the statement carries, so no caller text is spliced into the + statement. + """ + params = expr.get_params() + target = params.get("schema") or DEFAULT_SCHEMA + column_name = expr.column_name + counts = self._fn("COUNT_BIG", self._row_count_wildcard()) + + return self._select( + [ + self._col(column_name).as_("value"), + counts.as_("count"), + ], + from_=self._user_relation(expr.table_name, target), + group_by=[self._col(column_name)], + order_by=[ + (self._fn("COUNT_BIG", self._row_count_wildcard()), "DESC"), + (self._col(column_name), "ASC"), + ], + limit=int(expr.limit), + ) + + def _coverage_fraction(self, column, source, other_column, other_source): + """One side of the overlap: the share of *column* the other side holds. + + The numerator counts the *distinct* values of *column* that also occur + in *other_column*, as a ``COUNT(*)`` over a derived table of those + distinct values; the denominator counts *column*'s distinct non-NULL + values. Distinct on both sides is what makes the figure a coverage of + values rather than of rows -- a key repeated a thousand times that + matches once would otherwise read as well covered. + + ``IS NOT NULL`` is written on both halves rather than relied upon. In + the numerator it is load-bearing: a DISTINCT treats a NULL as one + value, so counting it would credit the tested side with a match it does + not have. In the divisor ``COUNT(DISTINCT ...)`` skips NULLs on its + own, but the statement says so explicitly, because the share is of the + values that could join and that is the fact being measured. + ``NULLIF(..., 0)`` then makes a column with no value at all answer NULL + rather than 0, which the parser reads as "no fraction": the difference + between "nothing matched" and "nothing to match" survives to the caller. + + The numerator is multiplied by ``1.0`` before it is divided, and that + is not decoration: SQL Server divides two integers with integer + division, so ``COUNT(*) / COUNT(DISTINCT ...)`` would answer 0 or 1 and + lose every fraction between them. The multiplier makes the division + decimal, which is exact for these counts rather than approximate. + """ + from rhosocial.activerecord.backend.expression import ( + FunctionCall, + InPredicate, + QueryStatement, + SelectModifier, + Subquery, + WhereClause, + ) + from rhosocial.activerecord.backend.expression.functions import ( + CountFunction, + ) + from rhosocial.activerecord.backend.expression.operators import ( + ArithmeticOperation, + ArithmeticOperator, + ) + from rhosocial.activerecord.backend.expression.sources import ( + DerivedTableSource, + ) + + dialect = self._node_dialect + # ``DISTINCT`` is a statement modifier rather than a predicate, so the + # overlap is built as a statement rather than through ``self._query``: + # a node carries every parameter it was given at construction and is + # never edited afterwards. + overlap = QueryStatement( + dialect, + select=[column], + from_=source, + where=WhereClause( + dialect, + self._and( + self._is_not_null(column), + InPredicate( + dialect, + column, + Subquery( + dialect, + self._query( + [other_column], + from_=other_source, + where=self._is_not_null(other_column), + ), + ), + ), + ), + ), + select_modifier=SelectModifier.DISTINCT, + ) + covered = self._subquery( + [self._fn("COUNT_BIG", self._row_count_wildcard())], + from_=DerivedTableSource(dialect, overlap, alias="covered"), + ) + total = self._subquery( + [ + FunctionCall( + dialect, + "NULLIF", + CountFunction(dialect, column, distinct=True), + self._lit(0, inline=True), + ) + ], + from_=source, + where=self._is_not_null(column), + ) + scaled = ArithmeticOperation( + dialect, + ArithmeticOperator.MUL, + covered, + self._lit(1.0, inline=True), + ) + return ArithmeticOperation( + dialect, ArithmeticOperator.DIV, scaled, total + ) + + def format_join_coverage_query_statement( + self, expr: "JoinCoverageQueryStatement" + ) -> Tuple[str, tuple]: + """Format a two-sided measurement of two columns' value domains. + + Unlike every other statement in this module, this one names two user + tables rather than one catalogue relation: the question is a relation + between two objects, so both sides are carried explicitly and their + identifiers are the dialect's to quote (see :meth:`_user_relation`). + The aliases ``l`` and ``r`` are what keep the two columns apart in the + rendered statement, and they are quoted like every other identifier. + + The row carries both fractions because neither is meaningful alone. A + left share of 1.0 beside a right share of 0.2 says every tested value + exists on the other side while most of that other domain is + unaccounted for; a single similarity score would average the two and + describe neither. A caller deciding whether a join is worth writing + needs to know which direction is incomplete. + + Both figures are counted over *distinct* values, with NULLs excluded + from the numerators and the denominators alike -- see + :meth:`_coverage_fraction` for both rules and for the ``1.0`` that + keeps SQL Server from answering an integer division. + + The cost is the two sides read for their distinct domains plus an + intersection of the two -- a double scan, and the reason the probe + belongs behind an explicit call rather than in any automatic pass. + + Collations are not reconciled. SQL Server compares the two columns + under whichever collation applies to the comparison, so values the + database would treat as equal may be counted as distinct here, and the + fractions are a *lower* bound across mismatched collations -- never an + upper one. + """ + params = expr.get_params() + target = params.get("schema") or DEFAULT_SCHEMA + + left_source = self._user_relation(params.get("left_table"), target, "l") + right_source = self._user_relation(params.get("right_table"), target, "r") + left_column = self._col(params.get("left_column"), "l") + right_column = self._col(params.get("right_column"), "r") + + return self._select( + [ + self._coverage_fraction( + left_column, left_source, right_column, right_source + ).as_("left_coverage"), + self._coverage_fraction( + right_column, right_source, left_column, left_source + ).as_("right_coverage"), + ] + ) + diff --git a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_cli_introspect.py b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_cli_introspect.py index ae166f01..10514684 100644 --- a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_cli_introspect.py +++ b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_cli_introspect.py @@ -20,7 +20,7 @@ from rhosocial.activerecord.backend.impl.sqlserver.cli import introspect as introspect_mod from rhosocial.activerecord.backend.impl.sqlserver.introspection import ( AsyncSQLServerIntrospector, - SyncSQLServerIntrospector, + SQLServerIntrospector, ) @@ -99,6 +99,76 @@ def table_exists(self, table_name, schema=None): def get_database_info(self): return {"db_name": "db", "version": "16.0.4250.1"} + # ---- the nine families shared with the MySQL and PostgreSQL CLIs ---- + # Each records the call the way the families above do, so a branch that + # names the wrong method, or drops an argument, shows up as a wrong + # recorded call rather than as a plausible listing. + + def list_unused_indexes(self, schema=None): + return self._record("list_unused_indexes", schema=schema) + + def list_table_usage_stats(self, schema=None): + return self._record("list_table_usage_stats", schema=schema) + + def list_runtime_metrics(self, schema=None): + return self._record("list_runtime_metrics", schema=schema) + + def list_index_usage_stats(self, schema=None): + return self._record("list_index_usage_stats", schema=schema) + + def list_object_dependencies(self, object_name=None, schema=None): + return self._record( + "list_object_dependencies", object_name=object_name, schema=schema + ) + + def get_ddl_statement(self, object_name, schema=None): + self.calls.append( + ("get_ddl_statement", {"object_name": object_name, "schema": schema}) + ) + return { + "schema": "dbo", + "object_type": "TABLE", + "object_name": object_name, + "ddl": "CREATE TABLE [users] ([id] int)", + "native": False, + } + + def list_column_data_profiles(self, table_name, schema=None): + return self._record("list_column_data_profiles", table_name=table_name, schema=schema) + + def sample_column_values(self, table_name, column_name, limit=100, schema=None): + return self._record( + "sample_column_values", + table_name=table_name, + column_name=column_name, + limit=limit, + schema=schema, + ) + + def estimate_join_coverage( + self, left_table, left_column, right_table, right_column, schema=None + ): + self.calls.append( + ( + "estimate_join_coverage", + { + "left_table": left_table, + "left_column": left_column, + "right_table": right_table, + "right_column": right_column, + "schema": schema, + }, + ) + ) + return { + "left_table": left_table, + "left_column": left_column, + "right_table": right_table, + "right_column": right_column, + "left_coverage": 1.0, + "right_coverage": 0.5, + } + def get_table_info(self, table_name, schema=None): self.calls.append(("get_table_info", {"table_name": table_name, "schema": schema})) return FakeTable(table_name) @@ -152,6 +222,39 @@ async def get_database_info(self): async def get_table_info(self, table_name, schema=None): return FakeIntrospector.get_table_info(self, table_name, schema) + async def list_unused_indexes(self, schema=None): + return FakeIntrospector.list_unused_indexes(self, schema) + + async def list_table_usage_stats(self, schema=None): + return FakeIntrospector.list_table_usage_stats(self, schema) + + async def list_runtime_metrics(self, schema=None): + return FakeIntrospector.list_runtime_metrics(self, schema) + + async def list_index_usage_stats(self, schema=None): + return FakeIntrospector.list_index_usage_stats(self, schema) + + async def list_object_dependencies(self, object_name=None, schema=None): + return FakeIntrospector.list_object_dependencies(self, object_name, schema) + + async def get_ddl_statement(self, object_name, schema=None): + return FakeIntrospector.get_ddl_statement(self, object_name, schema) + + async def list_column_data_profiles(self, table_name, schema=None): + return FakeIntrospector.list_column_data_profiles(self, table_name, schema) + + async def sample_column_values(self, table_name, column_name, limit=100, schema=None): + return FakeIntrospector.sample_column_values( + self, table_name, column_name, limit, schema + ) + + async def estimate_join_coverage( + self, left_table, left_column, right_table, right_column, schema=None + ): + return FakeIntrospector.estimate_join_coverage( + self, left_table, left_column, right_table, right_column, schema + ) + class FakeBackendBase: created = [] @@ -285,10 +388,16 @@ class TestDispatchContract: @pytest.mark.parametrize("kind", introspect_mod.INTROSPECT_TYPES) def test_type_maps_to_a_real_method_on_both_managers(self, kind): argv = ["introspect", kind] - if kind in ("columns", "indexes", "foreign-keys", "extended-properties"): + if kind in ("columns", "indexes", "foreign-keys", "extended-properties", + "ddl", "column-profiles", "value-samples"): argv.append("users") + if kind == "value-samples": + argv += ["--column", "status"] + if kind == "join-coverage": + argv += ["--left-table", "orders", "--left-column", "customer_id", + "--right-table", "customers", "--right-column", "id"] for method, _kwargs in introspect_mod._steps(make_args(argv)): - assert hasattr(SyncSQLServerIntrospector, method), method + assert hasattr(SQLServerIntrospector, method), method assert hasattr(AsyncSQLServerIntrospector, method), method def test_database_takes_no_object_name(self): @@ -318,14 +427,108 @@ class TestHandleDispatch: (["functions"], "list_functions"), (["sequences"], "list_sequences"), (["schemas"], "list_schemas"), + # The nine families shared with the MySQL and PostgreSQL CLIs. + # ``table-stats`` and ``ddl`` are refused by the dialect's own + # answer, so their rows below are what the branch *would* ask: + # the capability gate stops a real run before that, and the + # dedicated test file asserts both halves. + (["unused-indexes"], "list_unused_indexes"), + (["table-stats"], "list_table_usage_stats"), + (["runtime"], "list_runtime_metrics"), + (["index-stats"], "list_index_usage_stats"), + (["dependencies"], "list_object_dependencies"), + (["dependencies", "orders"], "list_object_dependencies"), + (["column-profiles", "orders"], "list_column_data_profiles"), + ( + ["value-samples", "orders", "--column", "status"], + "sample_column_values", + ), + ( + [ + "join-coverage", + "--left-table", "orders", + "--left-column", "customer_id", + "--right-table", "customers", + "--right-column", "id", + ], + "estimate_join_coverage", + ), ], ) def test_types_dispatch_and_render_json(self, argv, method, capsys): + if method == "estimate_join_coverage": + # The coverage answer is a record of its own, not the ``_record`` + # shape the other families return; the four names it carries are + # asserted by the test beside this one. + pytest.skip("asserted by test_join_coverage_receives_the_four_names") introspect_mod.handle(make_args(["introspect"] + argv + ["--database", "db"])) payload = json.loads(capsys.readouterr().out) assert isinstance(payload, list) assert payload[0]["method"] == method + def test_ddl_prints_the_engine_text_and_keeps_the_native_flag(self, capsys): + """A DDL text is a block, so the table format prints it as one; the + machine formats keep the record, ``native`` included.""" + introspect_mod.handle( + make_args(["introspect", "ddl", "users", "--database", "db", "-o", "json"]) + ) + payload = json.loads(capsys.readouterr().out) + assert payload[0]["object_name"] == "users" + assert payload[0]["ddl"] == "CREATE TABLE [users] ([id] int)" + assert payload[0]["native"] is False + + def test_join_coverage_receives_the_four_names(self, capsys): + introspect_mod.handle( + make_args([ + "introspect", "join-coverage", + "--left-table", "orders", + "--left-column", "customer_id", + "--right-table", "customers", + "--right-column", "id", + "--database", "db", + ]) + ) + method, kwargs = FakeIntrospector.instances[-1].calls[-1] + assert method == "estimate_join_coverage" + assert kwargs == { + "left_table": "orders", + "left_column": "customer_id", + "right_table": "customers", + "right_column": "id", + "schema": None, + } + payload = json.loads(capsys.readouterr().out) + assert payload[0]["left_coverage"] == 1.0 + assert payload[0]["right_coverage"] == 0.5 + + def test_a_coverage_the_server_cannot_measure_is_an_error(self, monkeypatch, capsys): + """``None`` is "not measured", which is not a zero-overlap answer.""" + + class UnmeasurableIntrospector(FakeIntrospector): + def estimate_join_coverage(self, *args, **kwargs): + return None + + monkeypatch.setattr( + introspect_mod, + "SQLServerBackend", + _tracking_class( + FakeBackendBase, "UnmeasurableBackend", UnmeasurableIntrospector + ), + ) + with pytest.raises(SystemExit) as exc: + introspect_mod.handle( + make_args([ + "introspect", "join-coverage", + "--left-table", "orders", + "--left-column", "customer_id", + "--right-table", "customers", + "--right-column", "id", + "--database", "db", + ]) + ) + assert exc.value.code == 1 + assert "could not measure" in capsys.readouterr().err + def test_tables_receives_include_system_and_no_schema(self, capsys): introspect_mod.handle( make_args(["introspect", "tables", "--database", "db", "--include-system"]) @@ -481,7 +684,65 @@ def test_the_advertised_list_is_the_capability_table(self): assert list(introspect_mod._TYPE_CAPABILITIES) == introspect_mod.INTROSPECT_TYPES def test_every_declared_type_has_a_name_rule(self): - assert set(introspect_mod._NAME_RULES) == set(introspect_mod.INTROSPECT_TYPES) + """A type advertised without a rule is one whose NAME decision nobody has made. + + The set-level half of pinning the rule table; the contents are pinned + beside it, because a set nobody asserts the members of is still a set + whose answers can drift one at a time. + """ + assert set(introspect_mod._NAME_RULE) == set(introspect_mod.INTROSPECT_TYPES) + + def test_the_table_says_what_each_types_name_means(self): + """The rule table as data, so editing one type is a visible change. + + ``join-coverage`` is the entry that reads as an outlier and is not: the + two tables it compares are named on flags, so a positional NAME would + be the third name for a question that already has its two -- and a + name the branch drops. + """ + assert introspect_mod._NAME_RULE == { + "tables": "forbidden", + "views": "optional", + "columns": "required", + "indexes": "required", + "foreign-keys": "required", + "extended-properties": "required", + "triggers": "optional", + "database": "forbidden", + "procedures": "forbidden", + "functions": "forbidden", + "sequences": "forbidden", + "schemas": "forbidden", + "unused-indexes": "forbidden", + "table-stats": "forbidden", + "runtime": "forbidden", + "index-stats": "forbidden", + "dependencies": "optional", + "ddl": "required", + "column-profiles": "required", + "value-samples": "required", + "join-coverage": "forbidden", + } + + def test_the_required_types_are_the_table_scoped_ones(self): + """The rule table and the branches agree on which types need a NAME. + + ``_steps`` consumes the name rather than re-deciding, so the set it + consumes one for is exactly the set the table marks ``required`` -- + one statement, read by the validator and by the dispatcher alike. + """ + required = { + kind for kind, rule in introspect_mod._NAME_RULE.items() if rule == "required" + } + assert required == { + "columns", + "indexes", + "foreign-keys", + "extended-properties", + "ddl", + "column-profiles", + "value-samples", + } def test_the_removed_aggregate_is_not_readvertised(self): assert "table" not in introspect_mod.INTROSPECT_TYPES @@ -496,7 +757,41 @@ def test_every_switch_is_declared_by_the_dialect(self, kind, switch): dialect = SQLServerDialect(version=(16, 0, 0)) answer = getattr(dialect, switch, None) assert callable(answer), f"{kind} names {switch}, which the dialect lacks" - assert answer() is True + + #: The two families SQL Server does not answer. The CLI offers them all + #: the same -- a type the engine refuses is refused at the capability + #: question with exit 2, not hidden from the parser -- so these are + #: asserted as the dialect's "no" rather than dropped from the table. + UNANSWERED_SWITCHES = { + "table-stats": "supports_table_stats", + "ddl": "supports_ddl_extraction", + } + + def test_the_two_unanswered_families_are_refused_by_the_dialect(self): + from rhosocial.activerecord.backend.impl.sqlserver.dialect import SQLServerDialect + + dialect = SQLServerDialect(version=(16, 0, 0)) + for kind, switch in self.UNANSWERED_SWITCHES.items(): + assert introspect_mod._TYPE_CAPABILITIES[kind] == switch + assert getattr(dialect, switch)() is False, kind + + def test_every_answered_family_answers_true(self): + """2022 is the newest release the matrix runs; every family the engine + answers must answer there. + + The version floors the DMV families carry -- 2005 for the index usage + view and the performance counters, 2008 for the dependency view, 2012 + for the pagination clause ``value-samples`` renders -- are all below + it, so a gate that refused here would be a gate set one release too + high. + """ + from rhosocial.activerecord.backend.impl.sqlserver.dialect import SQLServerDialect + + dialect = SQLServerDialect(version=(16, 0, 0)) + for kind, switch in introspect_mod._TYPE_CAPABILITIES.items(): + if kind in self.UNANSWERED_SWITCHES: + continue + assert getattr(dialect, switch)() is True, kind def test_help_lists_every_declared_type(self, capsys): parser = argparse.ArgumentParser() @@ -536,10 +831,14 @@ def test_a_caller_is_not_told_nothing(self): assert self._scopes() def test_every_declared_switch_answers_true(self): + """Kept for the families the engine answers; the two it does not are + asserted False in :class:`TestDeclaredVocabulary` instead.""" from rhosocial.activerecord.backend.impl.sqlserver.dialect import SQLServerDialect dialect = SQLServerDialect(version=(16, 0, 0)) - for switch in introspect_mod._TYPE_CAPABILITIES.values(): + for kind, switch in introspect_mod._TYPE_CAPABILITIES.items(): + if kind in TestDeclaredVocabulary.UNANSWERED_SWITCHES: + continue assert getattr(dialect, switch)() is True, switch @@ -598,6 +897,52 @@ def test_the_optional_names_are_not_refused(self): ) assert FakeIntrospector.instances[-1].calls[-1][0] == "list_triggers" + @pytest.mark.parametrize( + "kind", + sorted( + k for k, rule in introspect_mod._NAME_RULE.items() if rule == "forbidden" + ), + ) + def test_every_forbidden_type_is_refused_by_the_table(self, kind, capsys): + """Each category driven through the one validator that reads the table. + + The per-branch copies are gone, so this is the behaviour of the table + itself rather than of a branch that happens to remember. + """ + with pytest.raises(SystemExit) as exc: + introspect_mod.handle( + make_args(["introspect", kind, "users", "--database", "db"]) + ) + assert exc.value.code == 2 + assert f"Error: {kind} introspection takes no name" in capsys.readouterr().err + + @pytest.mark.parametrize( + "kind", + sorted( + k for k, rule in introspect_mod._NAME_RULE.items() if rule == "required" + ), + ) + def test_every_required_type_is_required_by_the_table(self, kind, capsys): + """A missing required NAME is an incomplete invocation, exit 1 -- and + the question is never asked, so a ``None`` cannot reach the catalogue. + """ + with pytest.raises(SystemExit) as exc: + introspect_mod.handle(make_args(["introspect", kind, "--database", "db"])) + assert exc.value.code == 1 + assert "Table name is required" in capsys.readouterr().err + + @pytest.mark.parametrize( + "kind", + sorted( + k for k, rule in introspect_mod._NAME_RULE.items() if rule == "optional" + ), + ) + def test_every_optional_type_may_be_given_a_name_or_not(self, kind): + introspect_mod.handle(make_args(["introspect", kind, "--database", "db"])) + assert FakeIntrospector.instances[-1].calls + introspect_mod.handle(make_args(["introspect", kind, "users", "--database", "db"])) + assert FakeIntrospector.instances[-1].calls + class TestHandleLifecycleAndConfig: def test_connection_config_resolved_from_args(self): diff --git a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_cli_status.py b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_cli_status.py index 3c6e0731..69dec32c 100644 --- a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_cli_status.py +++ b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_cli_status.py @@ -231,7 +231,7 @@ def test_type_dispatches_to_introspector_method( self, fake_backend_cls, monkeypatch, capsys, status_type, method ): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) RecordingIntrospector.instances = [] @@ -241,7 +241,7 @@ def test_type_dispatches_to_introspector_method( def test_overview_dispatch_and_json_payload(self, fake_backend_cls, monkeypatch, capsys): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) RecordingIntrospector.instances = [] @@ -256,7 +256,7 @@ def test_csv_tsv_fall_back_to_json_for_overview( self, fake_backend_cls, monkeypatch, capsys, fmt ): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) status_mod.handle(make_args(["status", "all", "--database", "db", "-o", fmt])) @@ -268,7 +268,7 @@ def test_table_format_without_rich_renders_json( ): assert status_mod.RICH_AVAILABLE is False # rich not installed in this env monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) status_mod.handle( @@ -281,7 +281,7 @@ def test_config_csv_output_writes_header_row( self, fake_backend_cls, monkeypatch, capsys ): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) status_mod.handle(make_args(["status", "config", "--database", "db", "-o", "csv"])) @@ -290,7 +290,7 @@ def test_config_csv_output_writes_header_row( def test_disconnect_called_in_finally(self, fake_backend_cls, monkeypatch, capsys): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) status_mod.handle(make_args(["status", "users", "--database", "db"])) @@ -298,7 +298,7 @@ def test_disconnect_called_in_finally(self, fake_backend_cls, monkeypatch, capsy def test_connection_config_resolved_from_args(self, fake_backend_cls, monkeypatch, capsys): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) argv = [ @@ -325,7 +325,7 @@ def test_ssl_matrix_without_explicit_flags( self, fake_backend_cls, monkeypatch, ssl_mode, expected_encrypt, expected_trust ): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) status_mod.handle( @@ -350,7 +350,7 @@ def test_explicit_flags_take_precedence_over_ssl_derivation( self, fake_backend_cls, monkeypatch, extra_args, expected_encrypt, expected_trust ): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) status_mod.handle(make_args(["status", "users", "--database", "db"] + extra_args)) @@ -360,7 +360,7 @@ def test_explicit_flags_take_precedence_over_ssl_derivation( def test_ssl_disabled_disables_encrypt(self, fake_backend_cls, monkeypatch): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) status_mod.handle( @@ -372,7 +372,7 @@ def test_ssl_disabled_disables_encrypt(self, fake_backend_cls, monkeypatch): def test_ssl_verify_full_requires_cert_check(self, fake_backend_cls, monkeypatch): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) status_mod.handle( @@ -384,7 +384,7 @@ def test_ssl_verify_full_requires_cert_check(self, fake_backend_cls, monkeypatch def test_trusted_connection_passthrough(self, fake_backend_cls, monkeypatch): monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", RecordingIntrospector + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector ) monkeypatch.setattr(status_mod, "SQLServerBackend", fake_backend_cls) status_mod.handle( @@ -408,12 +408,12 @@ def get_overview(self): raise introspector_exc monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", ExplodingIntrospector, ) else: monkeypatch.setattr( - f"{STATUS_INTROSPECTOR_MODULE}.SyncSQLServerStatusIntrospector", + f"{STATUS_INTROSPECTOR_MODULE}.SQLServerStatusIntrospector", RecordingIntrospector, ) with pytest.raises(SystemExit) as exc: diff --git a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_introspect_families_cli.py b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_introspect_families_cli.py new file mode 100644 index 00000000..b2e4d9ce --- /dev/null +++ b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_introspect_families_cli.py @@ -0,0 +1,739 @@ +# tests/rhosocial/activerecord_sqlserver_test/feature/backend/cli/test_introspect_families_cli.py +"""The nine families the MySQL and PostgreSQL CLIs ask, as this one asks them. + +No server is needed here. What is checked is the wiring the CLI owns: that +each type reaches its introspector method, that NAME and the type-specific +flags arrive as arguments rather than being parsed and dropped, and that a +family the connected server refuses is refused. Whether the answer is right +is the introspector's question, and the live tests carry it. + +Four of the nine are gated on the server's version because the SQL they +render needs an object a release can lack -- the dynamic management views +and the pagination clause -- so the floors are asserted on both sides: a +server at the floor is not refused, and a server below it is refused before +a statement is rendered rather than failing at execution. Two of the nine +this engine does not answer at all, and those are asserted as the dialect's +``False`` as well. +""" + +import argparse +import json +from typing import Optional + +import pytest + +from rhosocial.activerecord.backend.impl.sqlserver.cli import introspect as introspect_cli +from rhosocial.activerecord.backend.impl.sqlserver.dialect import SQLServerDialect +from rhosocial.activerecord.backend.impl.sqlserver.backend import async_backend as async_backend_mod +from rhosocial.activerecord.backend.introspection.types import ( + ColumnDataProfile, + ColumnValueFrequency, + DDLStatement, + JoinCoverage, +) + +#: The nine families this change wires, and the switch that gates each. The +#: dialect declares them, so a run against a real server passes the gate; the +#: refusal tests below turn the switch off to prove the gate is still read. +NEW_FAMILIES = { + "unused-indexes": "supports_unused_indexes_detection", + "table-stats": "supports_table_stats", + "runtime": "supports_runtime_stats", + "index-stats": "supports_index_stats", + "dependencies": "supports_object_dependencies", + "ddl": "supports_ddl_extraction", + "column-profiles": "supports_column_data_profile", + "value-samples": "supports_column_value_sample", + "join-coverage": "supports_join_coverage", +} + +#: The four families that answer for the whole database and take no NAME. +WHOLE_DATABASE_FAMILIES = ( + "unused-indexes", + "table-stats", + "runtime", + "index-stats", +) + +#: The releases the CI matrix runs, and the product version each reports. +SQL_2017 = (14, 0, 1000) +SQL_2019 = (15, 0, 2000) +SQL_2022 = (16, 0, 1000) + +#: The floor each gated family carries, and the reason for it. +VERSION_FLOORS = { + "unused-indexes": ((9, 0, 0), "sys.dm_db_index_usage_stats arrived with 2005"), + "index-stats": ((9, 0, 0), "sys.dm_db_index_usage_stats arrived with 2005"), + "runtime": ((9, 0, 0), "sys.dm_os_performance_counters arrived with 2005"), + "dependencies": ((10, 0, 0), "sys.sql_expression_dependencies arrived with 2008"), + "value-samples": ((11, 0, 0), "OFFSET ... FETCH is 2012 syntax"), +} + +#: A version below every floor: SQL Server 2000, which none of this is. +PRE_FLOOR = (8, 0, 0) + + +class _RecordingIntrospector: + """Records each introspection call, and returns what the test declared. + + ``__getattr__`` is the whole implementation: a test names the method once, + in the assertion, and a branch that reaches a different one fails on the + recorded name rather than on an attribute error that would hide the typo. + """ + + def __init__(self, results: Optional[dict] = None) -> None: + self.results = results or {} + self.calls = [] + + def __getattr__(self, name: str): + def call(*args, **kwargs): + self.calls.append((name, args, kwargs)) + return self.results.get(name) + + return call + + +class _AsyncRecordingIntrospector(_RecordingIntrospector): + """The async shape of the same recorder: methods are awaited, calls are not.""" + + def __getattr__(self, name: str): + async def call(*args, **kwargs): + self.calls.append((name, args, kwargs)) + return self.results.get(name) + + return call + + +def _args(type_: str, name: Optional[str] = None, **overrides) -> argparse.Namespace: + """The attributes a parsed invocation carries, with sane defaults.""" + values = dict( + output="json", + rich_ascii=False, + type=type_, + name=name, + database="mydb", + is_async=False, + named_connection=None, + include_system=False, + column=None, + limit=None, + left_table=None, + left_column=None, + right_table=None, + right_column=None, + ) + values.update(overrides) + return argparse.Namespace(**values) + + +class _AnsweringDialect(SQLServerDialect): + """SQL Server's dialect with the two unanswered probes turned on. + + The branch tests drive the dispatch, not the engine's answer, so the + switches the real dialect refuses are answered here. That this is a + *different* dialect from the one a run meets is the point: + :class:`TestTheTwoUnansweredFamilies` below asserts what the real one + says. + """ + + def supports_table_stats(self) -> bool: + return True + + def supports_ddl_extraction(self) -> bool: + return True + + +#: The version the stub's ``introspect_and_adapt()`` claims. 2022, the newest +#: release the matrix runs and the one every answered family must answer on. +STUB_SERVER_VERSION = SQL_2022 + + +def _run_sync(monkeypatch, introspector, args) -> None: + """Drive the sync chain against a stub whose dialect answers yes. + + The stub installs its dialect in ``introspect_and_adapt`` the way the real + backend does, so the capability question is answered from an adapted + dialect rather than from a bare one. + """ + class _Backend: + _connection = None + + def __init__(self, **_kwargs) -> None: + self.introspector = introspector + + def connect(self) -> None: + pass + + def introspect_and_adapt(self) -> None: + self.dialect = _AnsweringDialect(version=STUB_SERVER_VERSION) + + def disconnect(self) -> None: + pass + + monkeypatch.setattr(introspect_cli, "SQLServerBackend", _Backend) + monkeypatch.setattr( + introspect_cli, "resolve_connection_config_from_args", lambda _args: None + ) + introspect_cli.handle(args) + + +def _run_async(monkeypatch, introspector, args) -> None: + class _Backend: + _connection = None + + def __init__(self, **_kwargs) -> None: + self.introspector = introspector + + async def connect(self) -> None: + pass + + async def introspect_and_adapt(self) -> None: + self.dialect = _AnsweringDialect(version=STUB_SERVER_VERSION) + + async def disconnect(self) -> None: + pass + + # The command builds the async backend inside ``handle()``, so the name + # it resolves there is the one patched -- not a module attribute of this + # module. + monkeypatch.setattr(async_backend_mod, "AsyncSQLServerBackend", _Backend) + monkeypatch.setattr( + introspect_cli, "resolve_connection_config_from_args", lambda _args: None + ) + args.is_async = True + introspect_cli.handle(args) + + +class TestTheSchemaWideFamilies: + """The counters and listings describe the database, so NAME is refused.""" + + @pytest.mark.parametrize( + "type_, method", + [ + ("unused-indexes", "list_unused_indexes"), + ("table-stats", "list_table_usage_stats"), + ("runtime", "list_runtime_metrics"), + ("index-stats", "list_index_usage_stats"), + ], + ) + def test_the_family_asks_its_method_for_the_whole_schema( + self, type_, method, monkeypatch + ): + introspector = _RecordingIntrospector({method: []}) + + _run_sync(monkeypatch, introspector, _args(type_)) + + assert introspector.calls == [(method, (), {"schema": None})] + + @pytest.mark.parametrize("type_", WHOLE_DATABASE_FAMILIES) + def test_a_name_is_refused(self, type_, capsys): + with pytest.raises(SystemExit) as excinfo: + introspect_cli._validate_name(_args(type_, "orders")) + + assert excinfo.value.code == 2 + assert "takes no name" in capsys.readouterr().err + + +class TestTheDependenciesBranch: + """One optional NAME narrows the graph; omitting it asks for all of it.""" + + def test_a_named_object_is_asked_for_by_name(self, monkeypatch): + introspector = _RecordingIntrospector({"list_object_dependencies": []}) + + _run_sync(monkeypatch, introspector, _args("dependencies", "posts")) + + assert introspector.calls == [ + ("list_object_dependencies", (), {"object_name": "posts", "schema": None}) + ] + + def test_an_unnamed_run_asks_for_the_whole_graph(self, monkeypatch): + introspector = _RecordingIntrospector({"list_object_dependencies": []}) + + _run_sync(monkeypatch, introspector, _args("dependencies")) + + assert introspector.calls == [ + ("list_object_dependencies", (), {"object_name": None, "schema": None}) + ] + + +class TestTheDdlBranch: + """The schema object, not a name, is what a caller parses.""" + + def _statement(self) -> DDLStatement: + return DDLStatement( + schema="dbo", + object_type="PROCEDURE", + object_name="usp_orders", + ddl="CREATE PROCEDURE usp_orders AS BEGIN SET NOCOUNT ON END", + native=False, + ) + + def test_the_table_is_asked_for_by_name(self, monkeypatch): + introspector = _RecordingIntrospector({"get_ddl_statement": self._statement()}) + + _run_sync(monkeypatch, introspector, _args("ddl", "orders")) + + assert introspector.calls == [ + ("get_ddl_statement", (), {"object_name": "orders", "schema": None}) + ] + + def test_the_engine_text_reaches_json_with_its_native_flag( + self, monkeypatch, capsys + ): + introspector = _RecordingIntrospector({"get_ddl_statement": self._statement()}) + + _run_sync(monkeypatch, introspector, _args("ddl", "usp_orders")) + + payload = json.loads(capsys.readouterr().out) + assert payload[0]["object_name"] == "usp_orders" + assert payload[0]["ddl"].startswith("CREATE PROCEDURE") + assert payload[0]["native"] is False + + def test_no_ddl_is_an_error_not_an_empty_listing(self, monkeypatch, capsys): + introspector = _RecordingIntrospector({"get_ddl_statement": None}) + + with pytest.raises(SystemExit) as excinfo: + _run_sync(monkeypatch, introspector, _args("ddl", "orders")) + + assert excinfo.value.code == 1 + assert "no DDL available" in capsys.readouterr().err + + +class TestTheColumnProfilesBranch: + """The aggregate label must reach the reader; the numbers are not a catalogue.""" + + def _profile(self) -> ColumnDataProfile: + return ColumnDataProfile( + schema="dbo", + table_name="orders", + column_name="status", + null_fraction=0.0, + distinct_count=3, + row_count=120, + source="aggregate", + ) + + def test_the_table_is_asked_for_by_name(self, monkeypatch, capsys): + introspector = _RecordingIntrospector( + {"list_column_data_profiles": [self._profile()]} + ) + + _run_sync(monkeypatch, introspector, _args("column-profiles", "orders")) + + assert introspector.calls == [ + ("list_column_data_profiles", (), {"table_name": "orders", "schema": None}) + ] + + def test_the_run_says_the_numbers_are_aggregated(self, monkeypatch, capsys): + """Exact and expensive, on stderr so the JSON on stdout stays parseable.""" + introspector = _RecordingIntrospector( + {"list_column_data_profiles": [self._profile()]} + ) + + _run_sync(monkeypatch, introspector, _args("column-profiles", "orders")) + + captured = capsys.readouterr() + assert "aggregate" in captured.err + assert "full scan" in captured.err + payload = json.loads(captured.out) + assert payload[0]["source"] == "aggregate" + + def test_the_note_is_not_mistaken_for_a_result_row(self, monkeypatch, capsys): + """The note belongs on stderr; a line on stdout would break every parser.""" + introspector = _RecordingIntrospector( + {"list_column_data_profiles": [self._profile()]} + ) + + _run_sync(monkeypatch, introspector, _args("column-profiles", "orders")) + + out = capsys.readouterr().out + assert out.lstrip().startswith("[") + + +class TestTheValueSamplesBranch: + """A table and a column, and a limit that the introspector receives.""" + + def test_the_table_column_and_limit_reach_the_introspector(self, monkeypatch): + introspector = _RecordingIntrospector( + {"sample_column_values": [ColumnValueFrequency(value="paid", count=7)]} + ) + + _run_sync( + monkeypatch, + introspector, + _args("value-samples", "orders", column="status", limit=20), + ) + + assert introspector.calls == [ + ( + "sample_column_values", + (), + { + "table_name": "orders", + "column_name": "status", + "limit": 20, + "schema": None, + }, + ) + ] + + def test_no_limit_is_sent_when_the_flag_is_omitted(self, monkeypatch): + introspector = _RecordingIntrospector({"sample_column_values": []}) + + _run_sync( + monkeypatch, introspector, _args("value-samples", "orders", column="status") + ) + + # Absent ``--limit`` means absent from the call: the introspector's own + # signature answers with its own default, and the CLI does not restate + # it. A number here would fail the day the signature's default moved. + assert "limit" not in introspector.calls[0][2] + + def test_a_missing_column_is_an_incomplete_invocation(self, capsys): + with pytest.raises(SystemExit) as excinfo: + introspect_cli._validate_options(_args("value-samples", "orders")) + + assert excinfo.value.code == 1 + assert "--column" in capsys.readouterr().err + + +class TestTheJoinCoverageBranch: + """Four names, both sides of the measurement, in the order they are asked.""" + + def _coverage(self) -> JoinCoverage: + return JoinCoverage( + left_schema="dbo", + left_table="orders", + left_column="customer_id", + right_schema="dbo", + right_table="customers", + right_column="id", + left_coverage=1.0, + right_coverage=0.75, + ) + + def test_the_four_names_reach_the_introspector(self, monkeypatch): + introspector = _RecordingIntrospector( + {"estimate_join_coverage": self._coverage()} + ) + + _run_sync( + monkeypatch, + introspector, + _args( + "join-coverage", + left_table="orders", + left_column="customer_id", + right_table="customers", + right_column="id", + ), + ) + + assert introspector.calls == [ + ( + "estimate_join_coverage", + (), + { + "left_table": "orders", + "left_column": "customer_id", + "right_table": "customers", + "right_column": "id", + "schema": None, + }, + ) + ] + + def test_both_fractions_are_reported(self, monkeypatch, capsys): + introspector = _RecordingIntrospector( + {"estimate_join_coverage": self._coverage()} + ) + + _run_sync( + monkeypatch, + introspector, + _args( + "join-coverage", + left_table="orders", + left_column="customer_id", + right_table="customers", + right_column="id", + ), + ) + + payload = json.loads(capsys.readouterr().out) + assert payload[0]["left_coverage"] == 1.0 + assert payload[0]["right_coverage"] == 0.75 + + def test_missing_names_are_listed_together(self, capsys): + """One run should name every absent flag, not the first one it met.""" + with pytest.raises(SystemExit) as excinfo: + introspect_cli._validate_options( + _args( + "join-coverage", + left_table="orders", + right_table="customers", + ) + ) + + assert excinfo.value.code == 1 + err = capsys.readouterr().err + assert "--left-column" in err + assert "--right-column" in err + + def test_an_unmeasurable_pair_is_an_error(self, monkeypatch, capsys): + """``None`` is "not measured", which is not a zero-overlap answer.""" + introspector = _RecordingIntrospector({"estimate_join_coverage": None}) + + with pytest.raises(SystemExit) as excinfo: + _run_sync( + monkeypatch, + introspector, + _args( + "join-coverage", + left_table="orders", + left_column="customer_id", + right_table="customers", + right_column="id", + ), + ) + + assert excinfo.value.code == 1 + assert "could not measure" in capsys.readouterr().err + + +class TestTheArgumentRules: + def test_a_flag_the_type_does_not_read_is_refused(self, capsys): + with pytest.raises(SystemExit) as excinfo: + introspect_cli._validate_options(_args("tables", column="status")) + + assert excinfo.value.code == 2 + assert "takes no --column" in capsys.readouterr().err + + def test_a_required_name_is_required_for_the_new_families(self, capsys): + for type_ in ("ddl", "column-profiles", "value-samples"): + with pytest.raises(SystemExit) as excinfo: + introspect_cli._validate_name(_args(type_)) + assert excinfo.value.code == 1 + assert "required" in capsys.readouterr().err + + +class TestTheCapabilityGate: + @pytest.mark.parametrize("type_, switch", list(NEW_FAMILIES.items())) + def test_a_family_the_server_refuses_exits_two(self, type_, switch, capsys): + class _Dialect: + pass + + setattr(_Dialect, switch, staticmethod(lambda: False)) + + class _Backend: + dialect = _Dialect() + + with pytest.raises(SystemExit) as excinfo: + introspect_cli._require_capability(_Backend(), type_) + + assert excinfo.value.code == 2 + assert "not supported by this server" in capsys.readouterr().err + + +class TestTheAsyncPath: + """The second dispatch chain must ask the same question the first one does.""" + + @pytest.mark.parametrize( + "type_, method, args_kwargs, expected", + [ + ("unused-indexes", "list_unused_indexes", {}, ((), {"schema": None})), + ( + "dependencies", + "list_object_dependencies", + {"name": "orders"}, + ((), {"object_name": "orders", "schema": None}), + ), + ( + "ddl", + "get_ddl_statement", + {"name": "orders"}, + ((), {"object_name": "orders", "schema": None}), + ), + ( + "column-profiles", + "list_column_data_profiles", + {"name": "orders"}, + ((), {"table_name": "orders", "schema": None}), + ), + ( + "value-samples", + "sample_column_values", + {"name": "orders", "column": "status", "limit": 5}, + ( + (), + { + "table_name": "orders", + "column_name": "status", + "limit": 5, + "schema": None, + }, + ), + ), + ( + "join-coverage", + "estimate_join_coverage", + { + "left_table": "orders", + "left_column": "customer_id", + "right_table": "customers", + "right_column": "id", + }, + ( + (), + { + "left_table": "orders", + "left_column": "customer_id", + "right_table": "customers", + "right_column": "id", + "schema": None, + }, + ), + ), + ], + ) + def test_the_async_branch_calls_the_same_method_with_the_same_arguments( + self, type_, method, args_kwargs, expected, monkeypatch + ): + introspector = _AsyncRecordingIntrospector({method: []}) + + _run_async(monkeypatch, introspector, _args(type_, **args_kwargs)) + + assert introspector.calls == [(method, expected[0], expected[1])] + + +class TestTheTypeSurfaceExtension: + def test_each_new_family_has_a_switch_the_dialect_declares(self): + for type_, switch in NEW_FAMILIES.items(): + assert introspect_cli._TYPE_CAPABILITIES[type_] == switch + assert hasattr(SQLServerDialect, switch) + + +class TestTheVersionFloors: + """The families whose SQL needs an object a release can lack. + + The dynamic management views arrived across three releases and the + pagination clause ``value-samples`` renders is 2012 syntax, so each probe + asks the version accessor rather than answering a bare ``True``. That is + what keeps the CLI from turning a definition-time question into a runtime + syntax error: a server below the floor is refused at the capability + question, and the introspector is never reached. + + Asserted on both sides of each floor, and on every release the matrix + runs -- 2017, 2019 and 2022 -- which are the versions this repository + actually tests against. + """ + + @pytest.mark.parametrize("type_, switch", list(NEW_FAMILIES.items())) + def test_a_gated_family_answers_at_every_matrix_release(self, type_, switch): + if type_ not in VERSION_FLOORS: + pytest.skip(f"{type_} is not gated on a version") + for version in (SQL_2017, SQL_2019, SQL_2022): + dialect = SQLServerDialect(version=version) + assert getattr(dialect, switch)() is True, (type_, version) + + @pytest.mark.parametrize("type_, switch", list(NEW_FAMILIES.items())) + def test_a_gated_family_refuses_below_its_floor(self, type_, switch): + if type_ not in VERSION_FLOORS: + pytest.skip(f"{type_} is not gated on a version") + floor, reason = VERSION_FLOORS[type_] + below = (floor[0] - 1, 0, 0) + dialect = SQLServerDialect(version=below) + assert getattr(dialect, switch)() is False, f"{type_}: {reason}" + + @pytest.mark.parametrize("type_", list(VERSION_FLOORS)) + def test_the_refusal_comes_before_the_statement_is_rendered(self, type_, capsys): + """The regression itself: the statement is never handed to the engine. + + A probe that answered ``True`` below the floor rendered its SQL and + reported the resulting ``ProgrammingError`` as a generic "Error during + introspection". Below the floor the introspector is not reached at all. + """ + floor = VERSION_FLOORS[type_][0] + below = (floor[0] - 1, 0, 0) + + class _Backend: + dialect = SQLServerDialect(version=below) + + def __init__(self): + self.introspector = _RecordingIntrospector() + + with pytest.raises(SystemExit) as excinfo: + introspect_cli._require_capability(_Backend(), type_) + + assert excinfo.value.code == 2 + assert "not supported by this server" in capsys.readouterr().err + + def test_an_unadapted_dialect_refuses_a_gated_family(self): + """An unconfirmed version must not read as a capability the server has. + + ``version_at_least`` answers without raising, so a probe guarded on it + can be called before the handshake -- and a dialect nobody has adapted + is one whose version nobody has confirmed. + """ + dialect = SQLServerDialect() + assert dialect.version_or_none is None + for type_, switch in NEW_FAMILIES.items(): + if type_ in VERSION_FLOORS: + assert getattr(dialect, switch)() is False, type_ + + +class TestTheTwoUnansweredFamilies: + """The two families this engine does not answer, on any version. + + Per-table usage counters: SQL Server's counters are per *index*, and + summing them per table would be a derived figure wearing the shape of a + measured one; inserts and deletes are not tracked at all. DDL retrieval: + the engine has no per-object call of the kind MySQL's ``SHOW CREATE + TABLE`` is, so answering would mean assembling a reconstruction and + labelling it native. Both probes answer ``False``, the CLI refuses at the + capability question, and the roundtrip matrix keeps its pins for the two + statement classes with these reasons. + """ + + UNANSWERED = { + "table-stats": "supports_table_stats", + "ddl": "supports_ddl_extraction", + } + + @pytest.mark.parametrize("type_, switch", list(UNANSWERED.items())) + def test_the_dialect_refuses_them(self, type_, switch): + for version in (SQL_2017, SQL_2019, SQL_2022): + dialect = SQLServerDialect(version=version) + assert getattr(dialect, switch)() is False, type_ + + @pytest.mark.parametrize("type_", list(UNANSWERED)) + def test_the_cli_offers_them_and_refuses_them(self, type_, capsys): + """Offered, not hidden: the parser accepts the type and the gate + refuses it, which is a different answer from an invalid choice.""" + from rhosocial.activerecord.backend.impl.sqlserver.__main__ import parse_args + + args = parse_args(["introspect", type_, "--database", "db"]) + assert args.type == type_ + + class _Backend: + dialect = SQLServerDialect(version=SQL_2022) + + def __init__(self): + self.introspector = _RecordingIntrospector() + + with pytest.raises(SystemExit) as excinfo: + introspect_cli._require_capability(_Backend(), type_) + + assert excinfo.value.code == 2 + assert "not supported by this server" in capsys.readouterr().err + + def test_an_empty_per_table_counter_list_would_not_be_an_answer(self): + """The refusal is the capability answer, not "this database has none". + + A probe that answered ``True`` and a backend that returned an empty + list would read as a measured zero, which is the wrong answer twice + over: the engine never measured, and the reader cannot tell. + """ + dialect = SQLServerDialect(version=SQL_2022) + assert dialect.supports_table_stats() is False + assert dialect.supports_ddl_extraction() is False diff --git a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/expression/test_expression_roundtrip_all.py b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/expression/test_expression_roundtrip_all.py index 45578dc4..916d59a2 100644 --- a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/expression/test_expression_roundtrip_all.py +++ b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/expression/test_expression_roundtrip_all.py @@ -109,7 +109,7 @@ import pytest -from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError +from rhosocial.activerecord.backend.dialect.exceptions import (ProtocolNotImplementedError, UnsupportedFeatureError) from rhosocial.activerecord.backend.expression.core import Column, Literal from rhosocial.activerecord.backend.expression.objects import ( Database, @@ -151,7 +151,10 @@ MergeAction, MergeActionType, ) -from rhosocial.activerecord.backend.expression.statements.dql import QueryStatement +from rhosocial.activerecord.backend.expression.statements.dql import ( + QueryStatement, + SetOperationStatement, +) from rhosocial.activerecord.backend.expression.types import IntegerType from rhosocial.activerecord.backend.expression.types.custom import CustomType from rhosocial.activerecord.backend.expression.uuid import UUIDConstantExpression @@ -297,6 +300,20 @@ def _one_column_query(dialect): ) +def _compound_arms(dialect): + """Two single-column SELECTs over two tables, no bind parameters.""" + return [ + QueryStatement( + dialect, select=[Column(dialect, "id")], from_=_table_obj(dialect) + ), + QueryStatement( + dialect, + select=[Column(dialect, "id")], + from_=_table_obj(dialect, "other"), + ), + ] + + def _integer_column(dialect, name="col"): """A column definition carrying a *dialect-bound* type. @@ -375,6 +392,15 @@ def register_specials(): right=QueryStatement(d, select=[Column(d, "id")]), ), ) + # The statement half of the same pair: the word-form enum refuses the + # guess's bare string at construction, and SQL Server spells a compound + # SELECT, so a factory turns it into an instance the matrix can assert. + register_special_constructor( + "statements.dql.SetOperationStatement", + lambda d: SetOperationStatement( + d, op="union_all", parts=_compound_arms(d) + ), + ) register_special_constructor( "advanced_functions.CaseExpression", @@ -1030,7 +1056,37 @@ def shortest_path(d): #: fails here instead of passing quietly -- which is the whole point, given that #: this matrix used to swallow every one of these. LEGITIMATE_NON_RENDERS = { - # ---- bases that name an expression category, not a renderable thing ---- + # ---- core data-profile families ----------------------------------------- + # The three rows that stood here are gone: this dialect now renders all of + # them. ``sys.stats`` keys its statistics by object rather than by column + # and publishes no per-column null fraction or distinct count, so every + # figure is aggregated from the rows and labelled ``aggregate``; see + # ``SQLServerIntrospectionMixin.supports_column_data_profile``. + # ---- core operational families this dialect does not answer ---- + # The four rows that stood here for the unused-index, index-counter, + # runtime and dependency families are gone: this dialect renders all four. + # The dynamic management views describe a live instance rather than a + # schema-scoped catalogue, and that dependency is carried by joining the + # DMV through sys.objects and sys.schemas -- the same construct the + # per-table storage read already uses -- so the schema filter the + # statement needs is applied where the statement is built. The version + # floor and the VIEW DATABASE STATE / VIEW SERVER STATE grants each family + # needs are stated on the probes themselves. + # Two families remain unanswered and keep a pin. TableUsageStats: + # SQL Server's counters are per *index*, and summing them per table would + # be a derived figure wearing the shape of a measured counter; inserts + # and deletes are not tracked at all. DDLStatement: the engine has no + # per-object DDL retrieval call -- OBJECT_DEFINITION holds a routine's + # body, not a CREATE TABLE -- so answering would mean assembling a + # reconstruction and labelling it native. + "rhosocial.activerecord.backend.expression.introspection.TableUsageStatsQueryStatement": ( + ProtocolNotImplementedError, 'does not implement ', + ), + "rhosocial.activerecord.backend.expression.introspection.DDLStatementQueryStatement": ( + ProtocolNotImplementedError, 'does not implement ', + ), + +# ---- bases that name an expression category, not a renderable thing ---- # Each of these is a base class that deliberately declares no `format_method`, # so `to_sql()` reports that there is nothing to dispatch. They are not # `inspect.isabstract` -- they are concrete enough to construct -- so the @@ -2272,3 +2328,71 @@ def test_coverage_report(self, sqlserver_dialect): for fqn in sorted(ARTIFACT_GAPS): channel, _type, _fragment = ARTIFACT_GAPS[fqn] print(f" artifact gap: {fqn} ({channel} channel)") + +# --------------------------------------------------------------------------- +# The set-operation statement, rendered +# --------------------------------------------------------------------------- +# +# The matrix says ``SetOperationStatement`` renders on SQL Server, which is a +# verdict about the dialect rather than a claim about the bytes, so the bytes +# are pinned here. SQL Server brackets its identifiers, which is the part of +# the pinned SQL that would change if quoting were ever handed to the node +# instead of staying in the dialect. The bind parameters are asserted +# positionally, because an ordering bug still renders valid SQL. + + +def test_set_operation_statement_renders_the_pinned_sql(sqlserver_dialect): + """``UNION ALL`` over two arms, with the bracketed identifiers this dialect + spells them with.""" + node = SetOperationStatement( + sqlserver_dialect, op="union_all", parts=_compound_arms(sqlserver_dialect) + ) + + rendered, params = node.to_sql() + + assert rendered == "SELECT [id] FROM [t] UNION ALL SELECT [id] FROM [other]" + assert params == () + + +def test_set_operation_statement_spells_the_bare_union(sqlserver_dialect): + """The word form alone decides the keyword, so ``union`` differs.""" + node = SetOperationStatement( + sqlserver_dialect, op="union", parts=_compound_arms(sqlserver_dialect) + ) + + rendered, params = node.to_sql() + + assert rendered == "SELECT [id] FROM [t] UNION SELECT [id] FROM [other]" + assert params == () + + +def test_set_operation_statement_parameters_concatenate_in_part_order( + sqlserver_dialect, +): + """Each arm's parameters arrive in the order the operator joins them.""" + dialect = sqlserver_dialect + parts = [ + QueryStatement( + dialect, + select=[Column(dialect, "id"), Literal(dialect, 1)], + from_=_table_obj(dialect), + where=ComparisonPredicate( + dialect, "=", Column(dialect, "id"), Literal(dialect, 5) + ), + ), + QueryStatement( + dialect, + select=[Column(dialect, "id"), Literal(dialect, 2)], + from_=_table_obj(dialect, "u"), + ), + ] + node = SetOperationStatement(dialect, op="union_all", parts=parts) + + rendered, params = node.to_sql() + + assert rendered == ( + "SELECT [id], ? FROM [t] WHERE [id] = ? " + "UNION ALL " + "SELECT [id], ? FROM [u]" + ) + assert params == (1, 5, 2) diff --git a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/test_introspection_data_profile.py b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/test_introspection_data_profile.py new file mode 100644 index 00000000..3ea0bfac --- /dev/null +++ b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/test_introspection_data_profile.py @@ -0,0 +1,345 @@ +# tests/rhosocial/activerecord_sqlserver_test/feature/backend/test_introspection_data_profile.py +"""The data-profile family, measured against a real server. + +Rendering these statements proves only that they are well-formed. What needs +checking is whether SQL Server accepts them, and whether the arithmetic the +parser performs -- ``null_count / row_count`` into a fraction, ``COUNT(DISTINCT +...)`` into a count, the two endpoints out of the union, the ``source`` label +carried through -- lands on values a hand count agrees with. That is only +answerable against a server. + +The table is a hundred rows, built so every figure is arithmetic rather than +guesswork: + + id INT PRIMARY KEY i, unique -> 100 distinct, no nulls + code NVARCHAR(8) NOT NULL "a"/"b"/"c" by i % 3 -> 3 distinct, no nulls + amount DECIMAL(10,2) i, unique -> 100 distinct, no nulls + note NVARCHAR(8) NULL "x" when i % 5 == 0, "y" when i % 5 == 1, + NULL otherwise -> 2 distinct, 60% null + +The size is modest on purpose: this backend aggregates the rows rather than +reading a histogram, so the answer does not depend on a sample being large. It +only has to be big enough that a duplicate silently dropped by ``COUNT(DISTINCT +...)``, or a NULL miscounted by the ``SUM(CASE ...)`` arm, would move a number +off its hand-counted value -- which a hundred rows does. + +Three shapes differ from PostgreSQL and are pinned here rather than left to be +discovered. First, SQL Server aggregates, so the profile is ``aggregate`` and +every figure is exact, while PostgreSQL reads ``pg_stats`` and answers +``estimate``. Second, the value listing is a ``GROUP BY`` over the column, and +SQL Server groups NULLs into a real row -- so a column containing NULLs has a +NULL entry in its listing, usually the most frequent one. Third, the endpoints +arrive as text: the union resolves one result type per column position by +data-type precedence, and an ``int`` beside a ``varchar`` would have the text +converted to a number and fail, so both endpoints are cast in the statement. +The figures are exact; only the Python type is the union's, which is why the +decimal endpoints are read through ``Decimal`` here. + +The non-ASCII test is not decoration. The profile's ``column_name`` is the one +value in the statement that is caller data rather than a label, and this +backend binds it as a parameter rather than writing a string literal: a +non-prefixed literal is typed with the database's code page, so an identifier +outside that page would come back as question marks. The test fails if that +ever regresses to a literal. +""" + +from decimal import Decimal + +import pytest + + +TABLE = "intro_profile_orders" +CUSTOMERS = "intro_profile_customers" +UNICODE_TABLE = "intro_profile_unicode" +CODES = ("a", "b", "c") + +ROWS = 100 + +#: ``note`` is NULL on 60 of the 100 rows, "x" on 20 and "y" on 20. +NULL_SHARE = 0.6 +NOTE_COUNT = 20 + + +def _rows(): + rows = [] + for i in range(1, ROWS + 1): + if i % 5 == 0: + note = "x" + elif i % 5 == 1: + note = "y" + else: + note = None + rows.append((i, CODES[i % 3], i, note)) + return rows + + +def _setup(backend): + for table in (TABLE, CUSTOMERS, UNICODE_TABLE): + backend.execute(f"DROP TABLE IF EXISTS [{table}]") + backend.execute( + f"CREATE TABLE [{TABLE}] (" + "[id] INT NOT NULL PRIMARY KEY, " + "[code] NVARCHAR(8) NOT NULL, " + "[amount] DECIMAL(10,2) NOT NULL, " + "[note] NVARCHAR(8) NULL" + ")" + ) + backend.execute_many( + f"INSERT INTO [{TABLE}] ([id], [code], [amount], [note]) " + "VALUES (?, ?, ?, ?)", + _rows(), + ) + # The right side of every coverage measurement: four codes and one NULL. + # The NULL is the point -- it must not count as a value of the domain, so + # right coverage is 3/4 and not 3/5. + backend.execute( + f"CREATE TABLE [{CUSTOMERS}] (" + "[id] INT NOT NULL PRIMARY KEY, " + "[code] NVARCHAR(8) NULL" + ")" + ) + backend.execute_many( + f"INSERT INTO [{CUSTOMERS}] ([id], [code]) VALUES (?, ?)", + [(1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, None)], + ) + + +def _teardown(backend): + for table in (TABLE, CUSTOMERS, UNICODE_TABLE): + backend.execute(f"DROP TABLE IF EXISTS [{table}]") + + +class TestColumnDataProfile: + """What the rows hold, counted by the server rather than read from a catalog.""" + + def test_profile_matches_a_hand_count(self, sqlserver_backend): + _setup(sqlserver_backend) + try: + profiles = { + profile.column_name: profile + for profile in sqlserver_backend.introspector.list_column_data_profiles( + TABLE + ) + } + assert set(profiles) == {"id", "code", "amount", "note"} + + # A unique key: 100 distinct values, none of them NULL. + assert profiles["id"].null_fraction == 0.0 + assert profiles["id"].distinct_count == ROWS + assert int(profiles["id"].min_value) == 1 + assert int(profiles["id"].max_value) == ROWS + + # Three values, none null. COUNT(DISTINCT ...) ignores NULL on its + # own, so the count agrees with the listing below. + assert profiles["code"].null_fraction == 0.0 + assert profiles["code"].distinct_count == len(CODES) + assert profiles["code"].min_value == "a" + assert profiles["code"].max_value == "c" + + # The endpoints arrive as text: the union needs one type per + # position, and an int beside a varchar would fail the conversion + # the other way round. The figure is exact; only the Python type + # is the union's, which is why this goes through Decimal. + assert profiles["amount"].null_fraction == 0.0 + assert profiles["amount"].distinct_count == ROWS + assert Decimal(str(profiles["amount"].min_value)) == Decimal("1") + assert Decimal(str(profiles["amount"].max_value)) == Decimal(ROWS) + + # 60 of 100 rows are NULL, so a profile that dropped the figure or + # clamped it would still look plausible. + assert profiles["note"].null_fraction == pytest.approx(NULL_SHARE) + assert profiles["note"].distinct_count == 2 + assert profiles["note"].min_value == "x" + assert profiles["note"].max_value == "y" + finally: + _teardown(sqlserver_backend) + + def test_every_profile_says_it_was_aggregated(self, sqlserver_backend): + """The label rides the row, so a consumer never has to guess. + + SQL Server's statistics are keyed by statistics object, not by column, + and carry no per-column null fraction or distinct count -- so every + figure here is counted, and the row says so in ``source``. There is no + ``collected_at``: an aggregate is current by construction, and + borrowing a statistics object's ``last_updated`` would make a fresh + count look like a stale estimate. + """ + _setup(sqlserver_backend) + try: + profiles = sqlserver_backend.introspector.list_column_data_profiles(TABLE) + assert profiles + assert {profile.source for profile in profiles} == {"aggregate"} + for profile in profiles: + assert profile.collected_at is None + assert profile.row_count == ROWS + finally: + _teardown(sqlserver_backend) + + def test_a_non_ascii_column_name_survives_the_round_trip(self, sqlserver_backend): + """The label is caller data, so it is bound rather than written as text. + + A string literal in T-SQL without the ``N`` prefix is typed with the + database's code page; on a Latin collation an identifier outside that + page arrives as question marks. Binding it as a parameter makes the + driver type it Unicode, so the name comes back as it was declared. + """ + _setup(sqlserver_backend) + backend = sqlserver_backend + backend.execute( + f"CREATE TABLE [{UNICODE_TABLE}] ([id] INT NOT NULL, [备注] NVARCHAR(8) NULL)" + ) + backend.execute_many( + f"INSERT INTO [{UNICODE_TABLE}] ([id], [备注]) VALUES (?, ?)", + [(1, "一"), (2, None)], + ) + try: + profiles = backend.introspector.list_column_data_profiles(UNICODE_TABLE) + names = {profile.column_name for profile in profiles} + assert names == {"id", "备注"} + note = next(p for p in profiles if p.column_name == "备注") + assert note.null_fraction == pytest.approx(0.5) + assert note.distinct_count == 1 + finally: + _teardown(backend) + + def test_empty_table_answers_with_zero_rows_and_no_fraction(self, sqlserver_backend): + """A profile of nothing is a row count of zero, not a fabricated 0.0. + + ``SUM`` over no rows is NULL, so ``null_count`` is NULL and the + fraction is not derived. Saying 0.0 would claim the column holds no + NULLs, which is not something an empty table establishes. + """ + sqlserver_backend.execute(f"DROP TABLE IF EXISTS [{TABLE}]") + sqlserver_backend.execute(f"CREATE TABLE [{TABLE}] ([id] INT NULL)") + try: + profiles = sqlserver_backend.introspector.list_column_data_profiles(TABLE) + assert len(profiles) == 1 + profile = profiles[0] + assert profile.column_name == "id" + assert profile.row_count == 0 + assert profile.distinct_count == 0 + assert profile.null_fraction is None + assert profile.min_value is None + assert profile.max_value is None + assert profile.source == "aggregate" + finally: + _teardown(sqlserver_backend) + + +class TestColumnValueSample: + """The grouped scan, which unlike PostgreSQL's list can contain NULL.""" + + def test_listing_is_ordered_by_frequency(self, sqlserver_backend): + _setup(sqlserver_backend) + try: + samples = sqlserver_backend.introspector.sample_column_values( + TABLE, "code", limit=10 + ) + # CODES[i % 3] puts "b" on 34 rows (i % 3 == 1) and "a"/"c" on 33 + # each, so the order is frequency first and then the tie-break by + # value -- which is why "a" precedes "c" despite equal counts. + assert [item.value for item in samples] == ["b", "a", "c"] + assert [item.count for item in samples] == [34, 33, 33] + # The engine reports a count, not a fraction. A formatter that + # divided for it would have had to assume the row count. + assert all(item.frequency is None for item in samples) + finally: + _teardown(sqlserver_backend) + + def test_limit_bounds_the_listing(self, sqlserver_backend): + _setup(sqlserver_backend) + try: + assert ( + len( + sqlserver_backend.introspector.sample_column_values( + TABLE, "code", limit=2 + ) + ) + == 2 + ) + finally: + _teardown(sqlserver_backend) + + def test_null_is_a_value_of_the_domain(self, sqlserver_backend): + """SQL Server groups NULLs, so the listing has a NULL row -- usually first. + + This is the shape that differs from PostgreSQL, whose most-common list + never contains NULL because ``null_frac`` holds that fact separately. + A caller reading a listing for "what is in this column" must therefore + not assume every entry is a value; the profile's ``null_fraction`` is + the cross-engine answer, and this test is what says so for SQL Server. + """ + _setup(sqlserver_backend) + try: + samples = sqlserver_backend.introspector.sample_column_values( + TABLE, "note", limit=10 + ) + # NULL at 60 rows, then "x" and "y" at 20 each. + assert samples[0].value is None + assert samples[0].count == ROWS - 2 * NOTE_COUNT + assert {item.value for item in samples if item.value is not None} == { + "x", + "y", + } + assert sum(item.count for item in samples) == ROWS + finally: + _teardown(sqlserver_backend) + + +class TestJoinCoverage: + """Overlap measured over distinct values, on both sides.""" + + def test_coverage_is_measured_over_distinct_values(self, sqlserver_backend): + """3 of orders' 3 codes exist among customers' 4 non-null codes. + + Row counts would say something else entirely: 100 order rows map onto + 4 customer rows. Counting distinct values is what makes the number a + statement about the domains rather than about repetition, and the + ``1.0`` the statement multiplies by is what keeps SQL Server from + answering the integer division -- 0 -- instead. + """ + _setup(sqlserver_backend) + try: + coverage = sqlserver_backend.introspector.estimate_join_coverage( + TABLE, "code", CUSTOMERS, "code" + ) + assert coverage is not None + assert coverage.left_coverage == pytest.approx(1.0) + assert coverage.right_coverage == pytest.approx(0.75) + assert coverage.left_table == TABLE + assert coverage.right_table == CUSTOMERS + finally: + _teardown(sqlserver_backend) + + def test_the_measurement_is_not_symmetric(self, sqlserver_backend): + """Reversing the sides swaps the fractions, it does not copy them.""" + _setup(sqlserver_backend) + try: + coverage = sqlserver_backend.introspector.estimate_join_coverage( + CUSTOMERS, "code", TABLE, "code" + ) + assert coverage is not None + assert coverage.left_coverage == pytest.approx(0.75) + assert coverage.right_coverage == pytest.approx(1.0) + finally: + _teardown(sqlserver_backend) + + def test_an_empty_domain_is_no_measurement_rather_than_zero(self, sqlserver_backend): + """``NULLIF(..., 0)`` is the difference between "none" and "unmeasurable". + + With no customer rows there is nothing to divide by, and answering + ``0.0`` would read as "none of these values join" -- a different claim. + The left side still has a real denominator, so its 0.0 is a genuine + zero and must survive. + """ + _setup(sqlserver_backend) + try: + sqlserver_backend.execute(f"DELETE FROM [{CUSTOMERS}]") + coverage = sqlserver_backend.introspector.estimate_join_coverage( + TABLE, "code", CUSTOMERS, "code" + ) + assert coverage is not None + assert coverage.left_coverage == pytest.approx(0.0) + assert coverage.right_coverage is None + finally: + _teardown(sqlserver_backend) diff --git a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/test_introspection_sql_builders.py b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/test_introspection_sql_builders.py index 5793928e..5b267e41 100644 --- a/tests/rhosocial/activerecord_sqlserver_test/feature/backend/test_introspection_sql_builders.py +++ b/tests/rhosocial/activerecord_sqlserver_test/feature/backend/test_introspection_sql_builders.py @@ -45,12 +45,12 @@ SQLServerXmlType, ) from rhosocial.activerecord.backend.impl.sqlserver.introspection import ( - SyncSQLServerIntrospector, - SyncSQLServerStatusIntrospector, + SQLServerIntrospector, + SQLServerStatusIntrospector, ) -def make_introspector(version=(16, 0, 0)) -> SyncSQLServerIntrospector: +def make_introspector(version=(16, 0, 0)) -> SQLServerIntrospector: config = SQLServerConnectionConfig( host="localhost", port=1433, @@ -59,7 +59,7 @@ def make_introspector(version=(16, 0, 0)) -> SyncSQLServerIntrospector: password="", ) backend = SQLServerBackend(connection_config=config, version=version) - introspector = SyncSQLServerIntrospector(backend, executor=object()) + introspector = SQLServerIntrospector(backend, executor=object()) return introspector @@ -673,7 +673,7 @@ def test_null_extended_property_value_stays_none(self): assert (prop.class_id, prop.major_id, prop.minor_id) == (1, 77, 0) -def make_status_introspector(version=(16, 0, 0)) -> SyncSQLServerStatusIntrospector: +def make_status_introspector(version=(16, 0, 0)) -> SQLServerStatusIntrospector: config = SQLServerConnectionConfig( host="localhost", port=1433, @@ -682,7 +682,7 @@ def make_status_introspector(version=(16, 0, 0)) -> SyncSQLServerStatusIntrospec password="", ) backend = SQLServerBackend(connection_config=config, version=version) - return SyncSQLServerStatusIntrospector(backend) + return SQLServerStatusIntrospector(backend) class TestStatusQueries: @@ -941,3 +941,121 @@ def test_legacy_datetime_aliases_keep_their_bare_path(self): sdt = self._parse_one("smalldatetime", 0) assert sdt.data_type_full == "smalldatetime" assert sdt.parsed_data_type == DataType.parse_data_type_str(dialect, "smalldatetime") + +class TestTableStorageSlots: + """The per-table storage projection, and what a view answers. + + `sys.dm_db_partition_stats` holds one row per partition per index; the + listing reads it as three scalar subqueries rather than a GROUP BY so a + view -- which has no partition rows -- stays on the list with every + storage slot `None` instead of vanishing from it. + """ + + def _listing_sql(self, **kwargs) -> str: + introspector = make_introspector() + stmt = TableListQueryStatement(introspector._backend.dialect, **kwargs) + return introspector._backend.dialect.format_table_list_query_statement(stmt)[0] + + def test_probe_answers_true(self): + dialect = make_introspector()._backend.dialect + assert dialect.supports_table_storage_introspection() is True + + def test_storage_slots_ride_the_table_listing(self): + sql = self._listing_sql() + assert sql.count("[sys].[dm_db_partition_stats]") == 3 + assert "AS [size_bytes]" in sql + assert "AS [data_size_bytes]" in sql + assert "AS [index_size_bytes]" in sql + + def test_the_split_buckets_are_the_engine_s_own_partitions(self): + sql = self._listing_sql() + # index_id 0 (a heap) or 1 (a clustered index) is the table's own storage + assert "[ps].[index_id] IN (0, 1)" in sql + # everything above 1 is a nonclustered index's + assert "[ps].[index_id] > 1" in sql + # SQL Server pages are 8 KiB with no configuration switch + assert "* 8192 AS [size_bytes]" in sql + + def test_schema_stays_a_bind_parameter(self): + introspector = make_introspector() + stmt = TableListQueryStatement(introspector._backend.dialect, schema="sales") + _, params = introspector._backend.dialect.format_table_list_query_statement(stmt) + assert params == ("sales",) + + def test_view_rows_keep_empty_storage_slots(self): + """A view has no partitions: the subquery answers NULL, the slots stay + None -- "storage this engine does not track for this object" rather + than zero.""" + introspector = make_introspector() + rows = [ + {"TABLE_NAME": "t", "TABLE_TYPE": "U", "size_bytes": 24576, + "data_size_bytes": 20480, "index_size_bytes": 4096}, + {"TABLE_NAME": "v", "TABLE_TYPE": "V"}, + ] + by_name = {t.name: t for t in introspector.parse_tables(rows, "dbo")} + assert by_name["t"].size_bytes == 24576 + assert by_name["t"].data_size_bytes == 20480 + assert by_name["t"].index_size_bytes == 4096 + view = by_name["v"] + assert view.size_bytes is None + assert view.data_size_bytes is None + assert view.index_size_bytes is None + +class TestSequenceProjection: + """The own class keeps the full semantics; to_core carries the minimum.""" + + def test_every_common_field_projects(self): + from decimal import Decimal + + from rhosocial.activerecord.backend.impl.sqlserver.introspection.types import ( + SQLServerSequenceInfo, + ) + + own = SQLServerSequenceInfo( + name="seq_a", + schema="sales", + data_type="bigint", + start_value=Decimal("10"), + increment=Decimal("5"), + minimum_value=Decimal("1"), + maximum_value=Decimal("1000"), + current_value=Decimal("120"), + is_cycling=True, + cache_size=50, + ) + core = own.to_core() + assert core.name == "seq_a" + assert core.schema == "sales" + assert core.data_type == "bigint" + assert core.start_value == 10 + assert core.increment == 5 + assert core.minimum_value == 1 + assert core.maximum_value == 1000 + assert core.cycle is True + assert core.cache_size == 50 + assert core.last_value == 120 + + def test_non_integral_sequence_stays_complete_on_the_own_class(self): + """A sequence declared on decimal(10,2) has a non-integral increment: + the common minimum's integer field holds None for it, and the exact + Decimal stays where it is answerable -- on this class.""" + from decimal import Decimal + + from rhosocial.activerecord.backend.impl.sqlserver.introspection.types import ( + SQLServerSequenceInfo, + ) + + own = SQLServerSequenceInfo(name="seq_p", increment=Decimal("0.5")) + core = own.to_core() + assert core.increment is None + assert own.increment == Decimal("0.5") + + def test_unset_fields_project_as_none(self): + from rhosocial.activerecord.backend.impl.sqlserver.introspection.types import ( + SQLServerSequenceInfo, + ) + + core = SQLServerSequenceInfo(name="seq_b").to_core() + assert core.cycle is False + assert core.start_value is None + assert core.last_value is None diff --git a/tests/rhosocial/activerecord_sqlserver_test/feature/connection/conftest.py b/tests/rhosocial/activerecord_sqlserver_test/feature/connection/conftest.py index 60e1b027..2935c59d 100644 --- a/tests/rhosocial/activerecord_sqlserver_test/feature/connection/conftest.py +++ b/tests/rhosocial/activerecord_sqlserver_test/feature/connection/conftest.py @@ -5,8 +5,6 @@ This module provides fixtures for testing connection pools with SQL Server backends. """ -import asyncio -import sys import os from typing import Dict, Any, Generator @@ -24,16 +22,17 @@ ) -def ensure_compatible_event_loop(): - """Ensure event loop compatibility on Windows.""" - if sys.platform == "win32": - policy = asyncio.get_event_loop_policy() - if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): - new_policy = asyncio.WindowsSelectorEventLoopPolicy() - asyncio.set_event_loop_policy(new_policy) - - -ensure_compatible_event_loop() +# No event loop workaround is needed on Windows. This backend's async driver +# (aioodbc, which drives pyodbc) runs its work in a thread pool rather than +# registering the socket with the loop, so the platform's default +# ProactorEventLoop serves it fine; only a driver that registers the socket +# itself (the way psycopg does) requires a selector loop. A +# `WindowsSelectorEventLoopPolicy` switch used to sit here, copied from the +# postgres suite -- it is deprecated for removal in Python 3.16, and +# switching it would take asyncio subprocess and pipe I/O down with it. Do +# not reintroduce it: if a future driver needs a selector loop, follow the +# postgres suite's pytest_asyncio_loop_factories hook instead, which scopes +# the choice to the tests rather than the process. # --- Scenario Loading Logic ---