From 6e0d64836fbab89789fd1cbc22f1c0dd66d46e8a Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sat, 31 Oct 2026 21:06:39 +0800 Subject: [PATCH 01/27] feat(introspection)!: postgres takes its own status families home, and answers per-table storage The four status families that were only ever PostgreSQL's -- WAL, replication, archive, security -- leave core and come back as typed fields on PostgreSQLServerOverview(ServerOverview); get_overview() keeps its core signature because the subclass is a legal covariant return, and core's to_dict() iterates fields so the subclass's facts serialize without being re-declared. The status introspector, its CLI and the module exports follow the types home. Per-table storage, which the feedback named as missing, answers through the existing table listing: pg_relation_size, pg_indexes_size and pg_total_relation_size become expression nodes measured on the row's own oid, and the row count travels as two signals -- reltuples, the planner's estimate, preferred, and pg_stat_user_tables.n_live_tup, taken only when the planner has none (PostgreSQL 14+ reports -1 for a never-analyzed relation, which is the absence of an estimate rather than an empty table). The parse_row_count reducer normalizes -1 to None, keeps 0 as a real estimate, and invents nothing when neither source answers. supports_table_storage_introspection answers True: PostgreSQL's storage is row-plus-index shaped, so all four narrow slots of IntrospectionTableInfo are answerable, and that is what the CLI checks before offering the family. Version numbers in pyproject.toml are untouched on this branch; the release line takes the break at merge. CI: the core pin in test.yml points at feature/introspection-enhancement for this cross-repository round, because the changes here consume core APIs that exist only on that branch; move the pin back to the release line once core merges. The CLI capability contract caught the new probe with no type behind it: supports_table_storage_introspection answered True while the CLI offered no question that consulted it -- answerable but unaskable, which the rules forbid. PostgreSQL gets the type: table-storage asks for one named table and answers with the storage figures the listing already projects (total, data, index bytes and the row estimate), reading the listing once rather than the four questions a per-table info answer would spend. The test's fake rows gain the attribute access the real IntrospectionTableInfo already has. --- .github/workflows/test.yml | 8 +- .../backend/impl/postgres/cli/introspect.py | 48 +++- .../impl/postgres/expression/introspection.py | 140 ++++++++++- .../impl/postgres/introspection/__init__.py | 14 ++ .../postgres/introspection/introspector.py | 47 +++- .../introspection/status_introspector.py | 28 ++- .../postgres/introspection/status_types.py | 173 +++++++++++++ .../impl/postgres/mixins/introspection.py | 172 ++++++++++++- .../dialect/test_status_query_nodes.py | 63 +++++ .../test_table_storage_introspection.py | 227 ++++++++++++++++++ .../test_introspection_cli_capabilities.py | 23 +- .../introspection/test_status_introspector.py | 15 +- 12 files changed, 924 insertions(+), 34 deletions(-) create mode 100644 src/rhosocial/activerecord/backend/impl/postgres/introspection/status_types.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_table_storage_introspection.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 298859ee..76230519 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - name: Create postgres scenario config @@ -283,7 +283,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - name: Create postgres scenario config @@ -395,7 +395,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - name: Create postgres scenario config @@ -493,7 +493,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" - name: Create postgres scenario config diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py index de005806..4d72434a 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py @@ -49,6 +49,10 @@ "extensions": "supports_extension_introspection", "materialized-views": "supports_materialized_view_introspection", "sequences": "supports_sequence_introspection", + # The per-table storage family answers for one table: the three size + # figures ride the listing's projection, and this type asks it for one + # name rather than making the caller read the whole listing. + "table-storage": "supports_table_storage_introspection", } INTROSPECT_TYPES = list(_TYPE_CAPABILITIES) @@ -132,6 +136,9 @@ def create_parser(subparsers): %(prog)s sequences --database mydb %(prog)s sequences order_id_seq --database mydb + # Get one table's storage split (total, data, index bytes; row estimate) + %(prog)s table-storage orders --database mydb + # Output as JSON %(prog)s tables --database mydb -o json @@ -173,8 +180,9 @@ def create_parser(subparsers): "name", nargs="?", default=None, - help="Object name. Required for columns, indexes and foreign-keys " - "(the table they belong to); optional for triggers (a table) " + help="Object name. Required for columns, indexes, foreign-keys " + "(the table they belong to) and table-storage (the table itself); " + "optional for triggers (a table) " "and for views, materialized-views, sequences and extensions " "(one object); refused for tables and database", ) @@ -377,6 +385,26 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): sequences = introspector.list_sequences(schema=None) data = _serialize_for_output(sequences) provider.display_results(data, title="Sequences") + elif args.type == "table-storage": + # The storage figures ride the listing's projection, so the + # question asks the listing and picks the row -- one read, and + # none of the three per-table questions get_table_info would add + # (columns, indexes, foreign keys) that a storage answer does not + # need. The name is required because every figure is that + # table's own. + if not args.name: + print( + "Error: Table name is required for 'table-storage' introspection", + file=sys.stderr, + ) + sys.exit(1) + tables = introspector.list_tables(schema=None, include_system=args.include_system) + info = next((t for t in tables if t.name == args.name), None) + if info is None: + print(f"Error: Table '{args.name}' not found", file=sys.stderr) + sys.exit(1) + data = _serialize_for_output([info]) + provider.display_results(data, title=f"Table storage {args.name}") except ConnectionError as e: provider.display_connection_error(e) @@ -514,6 +542,22 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider sequences = await introspector.list_sequences(schema=None) data = _serialize_for_output(sequences) provider.display_results(data, title="Sequences") + elif args.type == "table-storage": + # The async twin of the sync branch: the same listing read, + # awaited, and the same pick. + if not args.name: + print( + "Error: Table name is required for 'table-storage' introspection", + file=sys.stderr, + ) + sys.exit(1) + tables = await introspector.list_tables(schema=None, include_system=args.include_system) + info = next((t for t in tables if t.name == args.name), None) + if info is None: + print(f"Error: Table '{args.name}' not found", file=sys.stderr) + sys.exit(1) + data = _serialize_for_output([info]) + provider.display_results(data, title=f"Table storage {args.name}") except ConnectionError as e: provider.display_connection_error(e) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/expression/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/expression/introspection.py index 3e9ab1ef..0e198dc5 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/expression/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/expression/introspection.py @@ -4,9 +4,10 @@ These expressions reuse the core parameter container contract (``IntrospectionExpression.get_params()``) and only redirect SQL generation to PostgreSQL-specific formatters, so the core introspection layer stays untouched. -The two ``pg_database_size`` nodes follow the same split for a query core has no -class for: the function node carries the call, the query node carries the row -shape the caller reads. +The ``pg_database_size`` pair and the ``pg_relation_size`` / +``pg_indexes_size`` trio follow the same split for questions core has no class +for: the function node carries the call, the query node carries the row shape +the caller reads. """ from typing import Optional, TYPE_CHECKING @@ -29,6 +30,9 @@ class for: the function node carries the call, the query node carries the row "PostgresSequenceListQueryStatement", "PostgresDatabaseSizeFunction", "PostgresDatabaseSizeQueryStatement", + "PostgresRelationSizeFunction", + "PostgresIndexesSizeFunction", + "PostgresTableStorageQueryStatement", ] @@ -228,3 +232,133 @@ def __init__( def format_method(self) -> str: """The dialect formatting method that renders this expression.""" return "format_database_size_query_statement" + + +class PostgresRelationSizeFunction(BaseExpression): + """The ``pg_relation_size(relid)`` call as a node of its own. + + PostgreSQL spells the on-disk size of one relation's heap as this + one-argument function. It is declared here rather than reached for through + the generic ``FunctionCall``, which core reserves for user-defined + functions: a standard function the server ships is a named node, so an + artifact naming it resolves to exactly one class instead of a catch-all + whose name is whatever the caller typed. + + The relation is named by its OID -- ``pg_class.oid``, the value every + catalog read of a table already carries -- and that number is a bind + parameter. PostgreSQL reads an ``oid`` where a ``regclass`` is expected + without a cast, which is why the argument binds bare; the point of binding + it is that a relation identifier is data and can never become part of the + SQL text. + + Example: + >>> expr = PostgresRelationSizeFunction(dialect, relid=16384) + >>> sql, params = expr.to_sql() + >>> sql + 'PG_RELATION_SIZE(%s)' + >>> params + (16384,) + """ + + def __init__( + self, + dialect: "SQLDialectBase", + relid: int, + ): + """ + Args: + dialect: The SQL dialect to use for formatting. + relid: OID of the relation whose heap is measured (``pg_class.oid``). + """ + super().__init__(dialect) + self.relid = relid + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_postgres_relation_size_function" + + +class PostgresIndexesSizeFunction(BaseExpression): + """The ``pg_indexes_size(relid)`` call as a node of its own. + + The index side of the same storage split ``pg_relation_size`` measures, and + a node for the same reason: a standard function the server ships is named + rather than spelled through the catch-all. Together with + :class:`PostgresRelationSizeFunction` it answers the two narrow slots of + :class:`~rhosocial.activerecord.backend.introspection.types.IntrospectionTableInfo` + -- ``data_size_bytes`` and ``index_size_bytes`` -- which + ``pg_total_relation_size`` cannot separate. + + The OID binds for the same reason the heap size's does: an identifier is + data, never text. + + Example: + >>> expr = PostgresIndexesSizeFunction(dialect, relid=16384) + >>> sql, params = expr.to_sql() + >>> sql + 'PG_INDEXES_SIZE(%s)' + >>> params + (16384,) + """ + + def __init__( + self, + dialect: "SQLDialectBase", + relid: int, + ): + """ + Args: + dialect: The SQL dialect to use for formatting. + relid: OID of the relation whose indexes are measured + (``pg_class.oid``). + """ + super().__init__(dialect) + self.relid = relid + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_postgres_indexes_size_function" + + +class PostgresTableStorageQueryStatement(BaseExpression): + """The one-row query that reports one relation's storage split. + + Wraps the two calls in the ``SELECT ... AS`` shape the storage slots name, + so a caller asks a node for the whole statement instead of assembling the + SELECT itself. The aliases are this query's contract: they are the field + names the parser fills, so the row shape and the record cannot drift apart. + + The total rides the table listing (``pg_total_relation_size``), and this + statement answers the split for one relation whose OID the caller already + holds -- the value every listing row carries. The OID binds once per call: + an identifier is data, and two bound copies of it cannot become SQL text. + + Example: + >>> expr = PostgresTableStorageQueryStatement(dialect, relid=16384) + >>> sql, params = expr.to_sql() + >>> sql + 'SELECT PG_RELATION_SIZE(%s) AS "data_size_bytes", PG_INDEXES_SIZE(%s) AS "index_size_bytes"' + >>> params + (16384, 16384) + """ + + def __init__( + self, + dialect: "SQLDialectBase", + relid: int, + ): + """ + Args: + dialect: The SQL dialect to use for formatting. + relid: OID of the relation whose storage is measured + (``pg_class.oid``). + """ + super().__init__(dialect) + self.relid = relid + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_table_storage_query_statement" diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/__init__.py index 94018285..e9b81692 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/__init__.py @@ -7,10 +7,24 @@ SyncPostgreSQLStatusIntrospector, AsyncPostgreSQLStatusIntrospector, ) +from .status_types import ( + ArchiveInfo, + PostgreSQLServerOverview, + ReplicationInfo, + SecurityInfo, + WALInfo, +) __all__ = [ "SyncPostgreSQLIntrospector", "AsyncPostgreSQLIntrospector", "SyncPostgreSQLStatusIntrospector", "AsyncPostgreSQLStatusIntrospector", + # PostgreSQL's own status families, taken back from core + "WALInfo", + "ReplicationInfo", + "ArchiveInfo", + "SecurityInfo", + # The overview that carries them + "PostgreSQLServerOverview", ] diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py index 968cd694..634acd68 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py @@ -151,6 +151,26 @@ def parse_tables(self, rows: List[Dict[str, Any]], schema: Optional[str]) -> Lis catalog distinguished them, and collapsing a materialized view or a partitioned table back into ``BASE_TABLE`` would lose that at the only layer that saw it. + + The storage slots are filled from the trio the listing projects: + ``size_bytes`` is the total footprint, ``data_size_bytes`` the heap and + ``index_size_bytes`` the index side of it -- the two PostgreSQL keeps + apart and a total cannot separate. A missing or unreadable cell is + reported as ``None`` rather than allowed to raise past the rest of the + answer, which is already known. + + **Which row estimate answers ``row_count``.** The listing carries two: + ``reltuples``, the planner's estimate, and ``live_row_count``, the + statistics view's ``n_live_tup``. The planner's is preferred, because it + is the number the server itself plans with and it is refreshed by + ``ANALYZE``/``VACUUM`` for exactly that purpose -- a value it holds is + the server's best current estimate of the relation's size. The + statistics view's is the fallback, taken only when the planner has + none: PostgreSQL 14+ reports ``-1`` for a relation that has never been + analyzed, which is the absence of an estimate rather than an empty + table, and in that case a live-row estimate says more than nothing does. + Neither is ever presented as exact, and neither is invented: a relation + with neither estimate reports ``None``. """ target_schema = schema if schema is not None else self.default_schema table_type_map = { @@ -166,13 +186,36 @@ def parse_tables(self, rows: List[Dict[str, Any]], schema: Optional[str]) -> Lis schema=target_schema, table_type=table_type_map.get(row.get("table_type", "BASE TABLE"), TableType.BASE_TABLE), comment=row.get("comment"), - # PostgreSQL 14+ returns -1 for unanalyzed tables, convert to None for consistency - row_count=row.get("row_count") if row.get("row_count") != -1 else None, + row_count=self.parse_row_count(row), size_bytes=row.get("size_bytes"), + data_size_bytes=row.get("data_size_bytes"), + index_size_bytes=row.get("index_size_bytes"), ) for row in rows ] + @staticmethod + def parse_row_count(row: Dict[str, Any]) -> Optional[int]: + """The row estimate a catalog row answers with, or ``None``. + + Two estimates arrive and one is chosen. ``reltuples`` is the planner's + and wins when it holds a value; ``n_live_tup`` (``live_row_count``) is + the statistics view's and answers only when the planner has none. + + PostgreSQL 14+ reports ``-1`` for a never-analyzed relation, which is + "no estimate" rather than "zero rows", so that value is normalized to + ``None`` here and the fallback is consulted; ``0`` from either source is + a real estimate and passes through unchanged. Anything unreadable is + ``None``: unknown is an answer, and guessing one would be worse. + """ + reltuples = row.get("row_count") + if reltuples is not None and reltuples != -1: + return int(reltuples) + live_rows = row.get("live_row_count") + if live_rows is not None and live_rows != -1: + return int(live_rows) + return None + def parse_columns(self, rows: List[Dict[str, Any]], table_name: str, schema: str) -> List[IntrospectionColumnInfo]: columns = [] for row in rows: diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_introspector.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_introspector.py index f77c9624..95136781 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_introspector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_introspector.py @@ -15,16 +15,11 @@ from rhosocial.activerecord.backend.introspection.status import ( StatusItem, StatusCategory, - ServerOverview, DatabaseBriefInfo, UserInfo, ConnectionInfo, StorageInfo, SessionInfo, - WALInfo, - ReplicationInfo, - ArchiveInfo, - SecurityInfo, ExtensionInfo, SyncAbstractStatusIntrospector, AsyncAbstractStatusIntrospector, @@ -52,6 +47,13 @@ PostgresArchiveStatsQuery, PostgresExtensionListQuery, ) +from .status_types import ( + ArchiveInfo, + PostgreSQLServerOverview, + ReplicationInfo, + SecurityInfo, + WALInfo, +) # PostgreSQL settings to include in status overview @@ -244,9 +246,15 @@ def _build_server_overview( archive: Optional[ArchiveInfo] = None, security: Optional[SecurityInfo] = None, extensions: Optional[List[ExtensionInfo]] = None, - ) -> ServerOverview: - """Build ServerOverview from collected data.""" - return ServerOverview( + ) -> PostgreSQLServerOverview: + """Build the PostgreSQL overview from collected data. + + The overview is a ``PostgreSQLServerOverview``: the four engine-level + families this server answers are typed fields on it, and ``get_overview`` + still returns what core declares because the subclass is a legal + covariant return. + """ + return PostgreSQLServerOverview( server_version=self._get_version_string(), server_vendor=self._get_vendor_name(), session=session, @@ -284,7 +292,7 @@ def _exec_query(self, sql: str, params: Optional[tuple] = None) -> List[Dict[str """Execute a SQL query synchronously.""" return self._backend.execute(sql, params).data or [] - def get_overview(self) -> ServerOverview: + def get_overview(self) -> PostgreSQLServerOverview: """Get complete PostgreSQL status overview.""" configuration = self.list_configuration() performance = self.list_performance_metrics() @@ -813,7 +821,7 @@ async def _exec_query_async(self, sql: str, params: Optional[tuple] = None) -> L result = await self._backend.execute(sql, params) return result.data if result else [] - async def get_overview(self) -> ServerOverview: + async def get_overview(self) -> PostgreSQLServerOverview: """Get complete PostgreSQL status overview.""" configuration = await self.list_configuration() performance = await self.list_performance_metrics() diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_types.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_types.py new file mode 100644 index 00000000..ac592b12 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_types.py @@ -0,0 +1,173 @@ +# src/rhosocial/activerecord/backend/impl/postgres/introspection/status_types.py +"""PostgreSQL's own server-status families, brought home from core. + +Core keeps the smallest shape an ActiveRecord layer generally needs, and +``ServerOverview`` is that shape: the public slots every engine can answer. The +four families below are not that -- a WAL is PostgreSQL's write mechanism, its +archiver and its hot-standby protocol answer questions no other engine asks -- +so they used to sit in core asserting that every backend had them. They now +live beside the introspector that fills them +(``...introspection.status_introspector``), and an overview that answers them is +the subclass declared at the bottom of this module. + +The types themselves are unchanged from where core carried them: the move is a +change of address, not of shape, so a caller that read ``wal.wal_level`` before +reads the same field now. + +Where the subclass fits: ``get_overview()``'s core signature does not change. +A ``PostgreSQLServerOverview`` *is* a ``ServerOverview``, so returning one from +the method core declares is a legal covariant return, and ``to_dict()`` +iterates the instance's fields instead of naming them, so the four facts this +subclass adds serialize without being re-declared anywhere. The LSP shape is +therefore: derive, add the typed slots, and the cross-dialect entry point keeps +working untouched (`.claude/rules/introspection.md` §5: one question, drifted +fields -- core holds the common minimum and each backend keeps its own complete +class beside it). +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + +from rhosocial.activerecord.backend.introspection.status import ServerOverview + +__all__ = [ + "WALInfo", + "ReplicationInfo", + "ArchiveInfo", + "SecurityInfo", + "PostgreSQLServerOverview", +] + + +@dataclass +class WALInfo: + """Write-Ahead Logging information. + + Attributes: + wal_level: WAL level (minimal, replica, logical) + wal_size_bytes: Current WAL size in bytes + wal_files: Number of WAL files + checkpoint_count: Number of checkpoints + checkpoint_time: Time of last checkpoint + wal_write_rate: WAL write rate in bytes/second + wal_segments: Current number of WAL segments + extra: Additional backend-specific information + """ + + wal_level: Optional[str] = None + wal_size_bytes: Optional[int] = None + wal_files: Optional[int] = None + checkpoint_count: Optional[int] = None + checkpoint_time: Optional[str] = None + wal_write_rate: Optional[float] = None + wal_segments: Optional[int] = None + extra: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ReplicationInfo: + """Replication status information. + + Attributes: + is_primary: Whether this server is a primary/master + is_standby: Whether this server is a standby/replica + replication_slots: Number of replication slots + active_replications: Number of active replication connections + lag_bytes: Replication lag in bytes (for standby) + streaming: Whether streaming replication is active + extra: Additional backend-specific information + """ + + is_primary: Optional[bool] = None + is_standby: Optional[bool] = None + replication_slots: Optional[int] = None + active_replications: Optional[int] = None + lag_bytes: Optional[int] = None + streaming: Optional[bool] = None + extra: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ArchiveInfo: + """Archive status information. + + Attributes: + archive_mode: Archive mode (on, off, always) + archive_command: Archive command + archive_timeout: Archive timeout in seconds + archived_count: Number of archived WAL files + failed_count: Number of failed archive attempts + last_archived: Time of last archive + last_failed: Time of last failed archive + extra: Additional backend-specific information + """ + + archive_mode: Optional[str] = None + archive_command: Optional[str] = None + archive_timeout: Optional[int] = None + archived_count: Optional[int] = None + failed_count: Optional[int] = None + last_archived: Optional[str] = None + last_failed: Optional[str] = None + extra: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class SecurityInfo: + """Security-related status information. + + Attributes: + ssl_enabled: Whether SSL is enabled + ssl_cert_file: SSL certificate file path + ssl_key_file: SSL key file path + ssl_ca_file: SSL CA file path + password_encryption: Password encryption method + auth_method: Authentication method + krb_enabled: Whether Kerberos is enabled + row_security: Row-level security status + extra: Additional backend-specific information + """ + + ssl_enabled: Optional[bool] = None + ssl_cert_file: Optional[str] = None + ssl_key_file: Optional[str] = None + ssl_ca_file: Optional[str] = None + password_encryption: Optional[str] = None + auth_method: Optional[str] = None + krb_enabled: Optional[bool] = None + row_security: Optional[bool] = None + extra: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class PostgreSQLServerOverview(ServerOverview): + """A complete PostgreSQL server status overview. + + The four typed slots are PostgreSQL's own answers to its own questions: + the WAL's state, the standby protocol, the archiver and the server's + security posture. They are named fields rather than ``extra`` entries + because a fact only one engine has is a typed field on that engine's class, + and because ``extra`` is the channel for a spelling that maps onto a core + member, never for a second family's fields. + + **Why this is a subclass and not a widened core type.** Core's + ``ServerOverview`` stays the cross-dialect entry point: it keeps the + public slots and declares nothing an engine cannot answer. This class + derives from it and adds what PostgreSQL can, so: + + * ``get_overview()``'s signature is unchanged -- the subclass is a legal + covariant return, and ``isinstance(overview, ServerOverview)`` still + holds for every caller written against core; + * ``to_dict()`` is field-driven, so ``wal``, ``replication``, ``archive`` + and ``security`` serialize without being re-declared here, exactly as + the fields inherited from core do. + + Every slot defaults to ``None``, so an overview built by a query that could + not answer one of the four families still describes the server honestly + instead of asserting a fact it does not have. + """ + + wal: Optional[WALInfo] = None + replication: Optional[ReplicationInfo] = None + archive: Optional[ArchiveInfo] = None + security: Optional[SecurityInfo] = None diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 7ee620c3..baed94ab 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -47,6 +47,9 @@ PostgresSequenceListQueryStatement, PostgresDatabaseSizeFunction, PostgresDatabaseSizeQueryStatement, + PostgresIndexesSizeFunction, + PostgresRelationSizeFunction, + PostgresTableStorageQueryStatement, ) from ..expression.status import ( PostgresSettingsQuery, @@ -124,6 +127,21 @@ def supports_trigger_introspection(self) -> bool: """PostgreSQL supports trigger introspection via pg_trigger.""" return True + def supports_table_storage_introspection(self) -> bool: + """PostgreSQL reports each table's storage split. + + PostgreSQL's storage is row-plus-index shaped, so the narrow slots of + ``IntrospectionTableInfo`` can all be answered: ``size_bytes`` from + ``pg_total_relation_size``, ``data_size_bytes`` from + ``pg_relation_size`` and ``index_size_bytes`` from ``pg_indexes_size``. + Declaring the family is what lets the table listing project the trio and + what a CLI checks before offering it; core defaults the probe to False + because an engine whose storage answers no part of that has nothing to + say, and an unanswered family must stay unoffered rather than answered + with an empty row. + """ + return True + def get_supported_introspection_scopes(self) -> List[IntrospectionScope]: """Get list of supported introspection scopes.""" return [ @@ -326,8 +344,41 @@ def _relkind_case(self, alias: str): alias=alias, ) + def _relation_size_call(self, function_name: str, alias: str): + """One of the three size calls, each measured on the relation's own OID. + + ``pg_total_relation_size`` reports the total footprint; + ``pg_relation_size`` and ``pg_indexes_size`` report the heap and the + index side of it -- the split the total cannot separate, which is why + all three are asked for rather than derived from one. The argument is + the same column in all three: the ``oid`` of the row being read. + """ + from rhosocial.activerecord.backend.expression import FunctionCall + + return FunctionCall(self, function_name, self._catalog_column("oid", "c"), alias=alias) + def _relation_projection(self) -> "list": - """The five columns a table listing and a single-table read both report.""" + """The columns a table listing and a single-table read both report. + + Three of them are the storage trio: ``size_bytes`` (the total, from + ``pg_total_relation_size``) with ``data_size_bytes`` and + ``index_size_bytes`` beside it. They are here because + :meth:`supports_table_storage_introspection` answers True for this + engine -- PostgreSQL's storage is row-plus-index shaped, so the narrow + slots of ``IntrospectionTableInfo`` carry the split instead of staying + empty. + + The row count travels as two signals, not one. ``reltuples`` is the + planner's estimate of the relation, maintained by ANALYZE; ``n_live_tup`` + is the statistics view's estimate of its live rows, maintained by the + stats collector. The two go stale by different routes and at different + times -- a freshly loaded table has neither, a table ANALYZEd long ago + may have a reliable ``reltuples`` and a decayed ``n_live_tup``, and a + table that has only ever been written through DML may have the reverse + -- so the projection carries both and the parser chooses. Both are cast + to ``bigint``, the type each is already stored as: the row shape states + it rather than leaving the driver to decide. + """ from rhosocial.activerecord.backend.expression import FunctionCall from rhosocial.activerecord.backend.expression.types import BigIntType @@ -338,12 +389,10 @@ def _relation_projection(self) -> "list": self, "obj_description", self._catalog_column("oid", "c"), alias="comment" ), self._catalog_column("reltuples", "c").cast(BigIntType(self)).as_("row_count"), - FunctionCall( - self, - "pg_total_relation_size", - self._catalog_column("oid", "c"), - alias="size_bytes", - ), + self._catalog_column("n_live_tup", "s").cast(BigIntType(self)).as_("live_row_count"), + self._relation_size_call("pg_total_relation_size", "size_bytes"), + self._relation_size_call("pg_relation_size", "data_size_bytes"), + self._relation_size_call("pg_indexes_size", "index_size_bytes"), ] def _pg_class_join(self): @@ -363,6 +412,28 @@ def _pg_class_join(self): ), ) + def _relation_source(self): + """``pg_class c JOIN pg_namespace n ... LEFT JOIN pg_stat_user_tables s ...``. + + The statistics join is what makes ``n_live_tup`` readable beside the + planner's ``reltuples``. It is a LEFT JOIN because that view lists only + the relations the stats collector has a row for: a relation it does not + list must still be reported, with the live-row column absent rather than + the whole row missing. + """ + from rhosocial.activerecord.backend.expression import ComparisonPredicate + + return self._pg_class_join().join( + self._catalog_relation("pg_stat_user_tables", "s"), + join_type="LEFT", + condition=ComparisonPredicate( + self, + "eq", + self._catalog_column("relid", "s"), + self._catalog_column("oid", "c"), + ), + ) + def _table_count_projection(self): """``table_count``: how many tables this database declares. @@ -557,7 +628,7 @@ def format_table_list_query_statement(self, expr: "TableListQueryStatement") -> return self._catalog_select( self._relation_projection(), - self._pg_class_join(), + self._relation_source(), conditions, [self._catalog_column("relname", "c")], ) @@ -581,7 +652,7 @@ def format_table_info_query_statement(self, expr: "TableInfoQueryStatement") -> return self._catalog_select( self._relation_projection(), - self._pg_class_join(), + self._relation_source(), [ ComparisonPredicate( self, "eq", self._catalog_column("nspname", "n"), self._catalog_value(schema) @@ -1830,6 +1901,89 @@ def format_database_size_query_statement( ], ).to_sql() + # ------------------------------------------------------------------ + # The per-table storage trio. The table listing asks the same three + # calls in its projection (``_relation_size_call``); these formatters + # answer them for one relation whose OID a caller already holds. + # ------------------------------------------------------------------ + + def format_postgres_relation_size_function( + self, expr: "PostgresRelationSizeFunction" + ) -> Tuple[str, tuple]: + """Format the ``pg_relation_size(relid)`` call. + + The relation's OID names a catalog row the caller already read, so it is + a bound value: PostgreSQL reads an ``oid`` where the function's + ``regclass`` argument is expected without a cast, and nothing about the + relation is spelled into the text. + + Args: + expr: Relation size function expression with the relation's OID. + + Returns: + Tuple of (SQL string, parameters tuple). + """ + from rhosocial.activerecord.backend.expression import FunctionCall + + params = expr.get_params() + return FunctionCall( + self, "pg_relation_size", self._catalog_value(params.get("relid")) + ).to_sql() + + def format_postgres_indexes_size_function( + self, expr: "PostgresIndexesSizeFunction" + ) -> Tuple[str, tuple]: + """Format the ``pg_indexes_size(relid)`` call. + + The index side of the split, bound for the same reason the heap side is: + the OID is data. Together they are the two narrow slots + ``pg_total_relation_size`` cannot separate. + + Args: + expr: Indexes size function expression with the relation's OID. + + Returns: + Tuple of (SQL string, parameters tuple). + """ + from rhosocial.activerecord.backend.expression import FunctionCall + + params = expr.get_params() + return FunctionCall( + self, "pg_indexes_size", self._catalog_value(params.get("relid")) + ).to_sql() + + def format_table_storage_query_statement( + self, expr: "PostgresTableStorageQueryStatement" + ) -> Tuple[str, tuple]: + """Format the query that reports one relation's storage split. + + The two calls in one statement, aliased to the slot names the parser + fills: the row shape is the contract, and asking both in one round trip + is what makes the split a single answer rather than two numbers read at + two different moments. The OID binds once per call. + + Args: + expr: Table storage query expression with the relation's OID. + + Returns: + Tuple of (SQL string, parameters tuple). + """ + from rhosocial.activerecord.backend.expression import FunctionCall, QueryStatement + + params = expr.get_params() + relid = params.get("relid") + return QueryStatement( + self, + select=[ + FunctionCall( + self, "pg_relation_size", self._catalog_value(relid), alias="data_size_bytes" + ), + FunctionCall( + self, "pg_indexes_size", self._catalog_value(relid), alias="index_size_bytes" + ), + ], + ).to_sql() + # ------------------------------------------------------------------ # Status and enum queries (org guide §16.2: the SQL text is the diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_status_query_nodes.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_status_query_nodes.py index ce09ecd3..d85209d7 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_status_query_nodes.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_status_query_nodes.py @@ -38,6 +38,14 @@ PostgresWalFileCountQuery, PostgresWalPositionQuery, ) +from rhosocial.activerecord.backend.impl.postgres.introspection.status_types import ( + ArchiveInfo, + PostgreSQLServerOverview, + ReplicationInfo, + SecurityInfo, + WALInfo, +) +from rhosocial.activerecord.backend.introspection.status import ServerOverview def _dialect(version=(17, 0, 0)): @@ -236,3 +244,58 @@ def test_role_list_reports_the_owner(self): assert "PG_CATALOG.PG_GET_USERBYID(oid)" in sql assert "FROM pg_roles ORDER BY rolname" in sql assert params == () + + +class TestOverviewSubclass: + """The four engine families ride this backend's own overview. + + Core keeps the public slots and no longer carries PostgreSQL's WAL, + replication, archive and security families; the subclass declares them as + typed fields, which is what lets ``get_overview()`` keep core's signature + while answering more than core's shape. + """ + + @staticmethod + def _overview(**overrides): + defaults = dict(server_version="17.4", server_vendor="PostgreSQL") + defaults.update(overrides) + return PostgreSQLServerOverview(**defaults) + + def test_the_subclass_is_the_core_entry_point(self): + overview = self._overview() + # Legal covariant return: a caller written against core's shape still + # accepts what this server answers. + assert isinstance(overview, ServerOverview) + + def test_the_families_are_typed_fields_that_default_to_none(self): + overview = self._overview() + assert overview.wal is None + assert overview.replication is None + assert overview.archive is None + assert overview.security is None + + def test_the_families_serialize_without_being_re_declared(self): + # to_dict() is field-driven, so the subclass's facts travel under their + # own names rather than being folded into ``extra``. + overview = self._overview( + wal=WALInfo(wal_level="replica"), + replication=ReplicationInfo(is_standby=True), + archive=ArchiveInfo(archive_mode="on"), + security=SecurityInfo(ssl_enabled=True), + ) + payload = overview.to_dict() + assert payload["wal"]["wal_level"] == "replica" + assert payload["replication"]["is_standby"] is True + assert payload["archive"]["archive_mode"] == "on" + assert payload["security"]["ssl_enabled"] is True + # The core slots are still there, unchanged. + assert payload["server_version"] == "17.4" + assert payload["databases"] == [] + + def test_the_taken_back_families_are_the_moved_types(self): + # The move changed the address, not the shape: these are the classes + # core used to carry, now imported from this backend. + assert WALInfo().wal_size_bytes is None + assert ReplicationInfo().lag_bytes is None + assert ArchiveInfo().archived_count is None + assert SecurityInfo().row_security is None diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_table_storage_introspection.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_table_storage_introspection.py new file mode 100644 index 00000000..192eedd8 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_table_storage_introspection.py @@ -0,0 +1,227 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_table_storage_introspection.py +"""The per-table storage trio (SQL generation and parsing only). + +A table's footprint is three numbers PostgreSQL keeps apart: the total, the +heap and the indexes. Core widened ``IntrospectionTableInfo`` with the two +narrow slots for exactly that, and this engine answers it -- so the listing +projects the trio, a caller can ask for one relation's split through a node, +and the parser fills the slots. + +Nothing here opens a connection: a statement is a rendered tree and a parser is +pure Python, so every case is a construction, render or parse assertion. The +live legs (a real server's numbers) belong to the introspection feature tests. +""" + +from types import SimpleNamespace + +from rhosocial.activerecord.backend.expression.introspection import TableListQueryStatement +from rhosocial.activerecord.backend.impl.postgres.dialect import PostgresDialect +from rhosocial.activerecord.backend.impl.postgres.expression.introspection import ( + PostgresIndexesSizeFunction, + PostgresRelationSizeFunction, + PostgresTableStorageQueryStatement, +) +from rhosocial.activerecord.backend.impl.postgres.introspection.introspector import ( + PostgreSQLIntrospectorMixin, +) + + +def _dialect(version=(17, 0, 0)): + return PostgresDialect(version=version) + + +class _FakeBackend: + """The two things a parser reads: the dialect and the connection config.""" + + def __init__(self, dialect, database="app"): + self.dialect = dialect + self.config = SimpleNamespace(database=database) + + +class _ParsingIntrospector(PostgreSQLIntrospectorMixin): + """An introspector with no executor: parsing never asks for one.""" + + def __init__(self, backend): + self._backend = backend + + +class TestSizeFunctionNodes: + """The two calls name their function and bind their relation.""" + + def test_relation_size_renders_the_call(self): + sql, params = PostgresRelationSizeFunction(_dialect(), relid=16384).to_sql() + assert sql == "PG_RELATION_SIZE(%s)" + assert params == (16384,) + + def test_indexes_size_renders_the_call(self): + sql, params = PostgresIndexesSizeFunction(_dialect(), relid=16384).to_sql() + assert sql == "PG_INDEXES_SIZE(%s)" + assert params == (16384,) + + def test_the_oid_travels_as_a_parameter_not_as_text(self): + # The relation is named by data: a bound OID cannot become SQL text, + # whatever number the caller holds, and the statement carries no + # literal at all in its place. + sql, params = PostgresRelationSizeFunction(_dialect(), relid=99_999_999).to_sql() + assert sql == "PG_RELATION_SIZE(%s)" + assert params == (99_999_999,) + + +class TestTableStorageQueryNode: + """The one-row statement reports the split under the slot names.""" + + def test_both_slots_are_asked_for_in_one_statement(self): + sql, params = PostgresTableStorageQueryStatement(_dialect(), relid=16384).to_sql() + assert sql == ( + 'SELECT PG_RELATION_SIZE(%s) AS "data_size_bytes", ' + 'PG_INDEXES_SIZE(%s) AS "index_size_bytes"' + ) + # The OID binds once per call: the same relation, two measurements. + assert params == (16384, 16384) + + def test_one_statement_rather_than_two(self): + sql, _ = PostgresTableStorageQueryStatement(_dialect(), relid=16384).to_sql() + assert sql.count("SELECT") == 1 + assert ";" not in sql + + +class TestListingProjection: + """The table listing asks the trio and reads the statistics view beside it.""" + + def _sql(self): + sql, params = TableListQueryStatement(_dialect(), schema="public").to_sql() + assert params == ("public",) + return sql + + def test_storage_trio_is_projected_from_the_relations_own_oid(self): + sql = self._sql() + assert 'PG_TOTAL_RELATION_SIZE(c.oid) AS "size_bytes"' in sql + assert 'PG_RELATION_SIZE(c.oid) AS "data_size_bytes"' in sql + assert 'PG_INDEXES_SIZE(c.oid) AS "index_size_bytes"' in sql + + def test_the_live_row_estimate_rides_a_left_join(self): + sql = self._sql() + assert "LEFT JOIN pg_stat_user_tables AS s ON s.relid = c.oid" in sql + assert 's.n_live_tup::BIGINT AS "live_row_count"' in sql + # The planner's estimate stays: the two are read side by side. + assert 'c.reltuples::BIGINT AS "row_count"' in sql + + def test_the_statistics_join_does_not_reach_the_view_listing(self): + # A view listing is a different question from a table listing, and it + # has no use for a row estimate or a storage split: the join belongs to + # the relation questions only. + from rhosocial.activerecord.backend.expression.introspection import ViewListQueryStatement + + sql, _ = ViewListQueryStatement(_dialect(), schema="public").to_sql() + assert "pg_stat_user_tables" not in sql + assert "n_live_tup" not in sql + + +class TestCapabilityProbe: + """The family is declared, because this engine can answer it.""" + + def test_the_probe_answers_true(self): + assert _dialect().supports_table_storage_introspection() is True + + def test_the_probe_is_the_core_contract(self): + # A probe core does not declare cannot be asked about one; this one + # answers for the engine rather than staying on core's default. + from rhosocial.activerecord.backend.dialect.mixins import IntrospectionMixin + + assert ( + IntrospectionMixin.supports_table_storage_introspection(IntrospectionMixin()) is False + ) + + +class TestRowCountContract: + """Which of the two estimates answers ``row_count`` is a named contract.""" + + @staticmethod + def _estimate(**overrides): + row = {"row_count": 42, "live_row_count": 40} + row.update(overrides) + return row + + def _choose(self, row): + return PostgreSQLIntrospectorMixin.parse_row_count(row) + + def test_the_planners_estimate_is_the_contract_that_answers(self): + assert self._choose(self._estimate(row_count=42, live_row_count=40)) == 42 + + def test_zero_is_an_estimate_not_an_absence(self): + assert self._choose(self._estimate(row_count=0, live_row_count=0)) == 0 + + def test_the_live_estimate_answers_when_the_planner_has_none(self): + assert self._choose(self._estimate(row_count=-1, live_row_count=17)) == 17 + + def test_neither_estimate_is_unknown(self): + assert self._choose(self._estimate(row_count=-1, live_row_count=None)) is None + assert self._choose(self._estimate(row_count=None, live_row_count=None)) is None + + +class TestStorageParsing: + """The parser fills the slots and reads the contract's answer.""" + + @staticmethod + def _row(**overrides): + row = { + "table_name": "orders", + "table_type": "BASE TABLE", + "comment": None, + "row_count": 42, + "live_row_count": 40, + "size_bytes": 8192, + "data_size_bytes": 6144, + "index_size_bytes": 2048, + } + row.update(overrides) + return row + + def _parse(self, rows): + return _ParsingIntrospector(_FakeBackend(_dialect())).parse_tables(rows, "public") + + def test_the_split_is_reported_beside_the_total(self): + table = self._parse([self._row()])[0] + assert table.size_bytes == 8192 + assert table.data_size_bytes == 6144 + assert table.index_size_bytes == 2048 + + def test_absent_split_stays_none(self): + # A relation whose size calls answer nothing reports unknown rather + # than zero: zero is a size, and this engine did not measure it. + table = self._parse( + [self._row(size_bytes=None, data_size_bytes=None, index_size_bytes=None)] + )[0] + assert table.size_bytes is None + assert table.data_size_bytes is None + assert table.index_size_bytes is None + + def test_the_planners_estimate_wins_when_it_has_one(self): + table = self._parse([self._row(row_count=42, live_row_count=40)])[0] + assert table.row_count == 42 + + def test_a_measured_zero_is_reported_rather_than_unknown(self): + # An analyzed empty table reports 0 from either source; that is a + # measured emptiness, not a missing column. + table = self._parse([self._row(row_count=0, live_row_count=0)])[0] + assert table.row_count == 0 + + def test_the_live_estimate_answers_when_the_planner_has_none(self): + # PostgreSQL 14+ reports -1 for a never-analyzed relation, which means + # "no estimate" and not "no rows": the statistics view's number then + # says more than nothing does. + table = self._parse([self._row(row_count=-1, live_row_count=17)])[0] + assert table.row_count == 17 + + def test_no_estimate_from_either_source_is_unknown(self): + table = self._parse([self._row(row_count=-1, live_row_count=None)])[0] + assert table.row_count is None + + def test_absent_statistics_column_is_unknown(self): + # A relation the stats view does not list still reports its other + # slots; only the row count goes unknown. + row = self._row() + del row["live_row_count"] + table = self._parse([row])[0] + assert table.row_count == 42 + assert table.data_size_bytes == 6144 diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_capabilities.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_capabilities.py index b01d91cc..fd06f26c 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_capabilities.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_capabilities.py @@ -44,6 +44,22 @@ def switch(): return switch +class _Row(dict): + """A fake catalogue row: a dict, and the attribute the real record has. + + The real ``list_tables`` answers ``IntrospectionTableInfo`` objects; the + fake answers dicts because the serializers read both. A consumer that + reads an attribute -- like the ``table-storage`` branch picking the named + row -- gets the real contract's surface rather than a surprise. + """ + + def __getattr__(self, item): + try: + return self[item] + except KeyError: + raise AttributeError(item) + + class FakeIntrospector: """One method per family, each recording the call it answered.""" @@ -58,7 +74,7 @@ def _record(self, name, *args): def list_tables(self, schema=None, include_system=False): self._record("list_tables", schema, include_system) - return [{"name": "users"}] + return [_Row(name="users")] def list_views(self, schema=None): self._record("list_views", schema) @@ -170,6 +186,10 @@ async def disconnect(self): "extensions": "list_extensions", "materialized-views": "list_materialized_views", "sequences": "list_sequences", + # The storage figures ride the listing's projection, so the CLI asks the + # listing and picks the row -- one read, not the four a per-table info + # answer would spend. + "table-storage": "list_tables", } #: The name each type needs to reach its branch; the family listings take none. @@ -177,6 +197,7 @@ async def disconnect(self): "columns": ["users"], "indexes": ["users"], "foreign-keys": ["users"], + "table-storage": ["users"], } diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_status_introspector.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_status_introspector.py index d247e128..1d05a758 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_status_introspector.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_status_introspector.py @@ -17,11 +17,14 @@ UserInfo, ConnectionInfo, StorageInfo, - WALInfo, - ReplicationInfo, + ExtensionInfo, +) +from rhosocial.activerecord.backend.impl.postgres.introspection.status_types import ( ArchiveInfo, + PostgreSQLServerOverview, + ReplicationInfo, SecurityInfo, - ExtensionInfo, + WALInfo, ) @@ -35,6 +38,9 @@ def test_get_overview(self, postgres_backend_single): overview = status.get_overview() assert isinstance(overview, ServerOverview) + # The engine-level families ride this backend's own subclass, so the + # cross-dialect entry point still accepts what PostgreSQL answers. + assert isinstance(overview, PostgreSQLServerOverview) assert overview.server_vendor == "PostgreSQL" assert overview.server_version is not None assert isinstance(overview.configuration, list) @@ -363,6 +369,9 @@ async def test_get_overview(self, async_postgres_backend_single): overview = await status.get_overview() assert isinstance(overview, ServerOverview) + # The engine-level families ride this backend's own subclass, so the + # cross-dialect entry point still accepts what PostgreSQL answers. + assert isinstance(overview, PostgreSQLServerOverview) assert overview.server_vendor == "PostgreSQL" assert overview.server_version is not None assert isinstance(overview.configuration, list) From 354050e3bcb243f08e2079c641e2e2ad37e769f5 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 1 Nov 2026 12:13:38 +0800 Subject: [PATCH 02/27] fix(postgres): run psycopg's async mode on the event loop it can wait on psycopg waits on the libpq socket with loop.add_reader/add_writer, which on Windows the platform default event loop (ProactorEventLoop, since 3.8) does not implement -- so every "await psycopg async under the default loop" path failed there. Measured on a Windows box: psycopg 3.3.6 refuses with "Psycopg cannot use the 'ProactorEventLoop' to run in async mode. Please use a compatible event loop, for instance by running 'asyncio.run(..., loop_factory=asyncio.SelectorEventLoop(selectors.SelectSelector()))'", and that suggested loop answers SELECT 42 against a live server. The product path gains one honest helper -- async_support.selector_event_loop()/run_coroutine(): the loop psycopg's async mode requires, constructed explicitly (asyncio.run's loop_factory argument is 3.13+, this backend still supports 3.8), with asyncio.run-grade lifecycle: cancel pending tasks on KeyboardInterrupt, shut async generators down, close and detach. CLI's three --async modes (introspect, query, status) run through it and now work on Windows out of the box; verified against a live server. The test path stops switching the global policy. Three conftests each set the deprecated WindowsSelectorEventLoopPolicy (announced for removal in Python 3.16, and a selector loop on Windows cannot serve asyncio subprocess or pipe I/O, so the switch could break what else runs in that process). They are replaced by pytest-asyncio 1.x's supported mechanism -- a module-level pytest_asyncio_loop_factories hook returning a selector loop, which scopes the choice to the tests' loops instead of the process. The async pool tests against a live server pass under it on Windows, and the deprecation warnings are gone. The unit tests assert the capability psycopg actually depends on (add_reader/add_writer on the constructed loop) rather than only a class name, plus the lifecycle: value propagation, exception propagation with the loop still closed, refusal inside a running loop, and interruption cancelling what is pending. AsyncPostgresBackend.connect() gains no guard of its own: psycopg 3.3+ already refuses with the fix named, and duplicating it would put the driver's evolution on this side of the seam. Version numbers in pyproject.toml are untouched on this branch. The first CI run caught the hook answering None on Linux: pytest-asyncio 1.4.1 requires a non-empty mapping from pytest_asyncio_loop_factories, so "nothing to do here" raised a UsageError in every async test's setup (2736 errors). The hook now exists only on Windows -- not defining it is how the hook says "no opinion", and everywhere else the default loop is already selector-based with the async tests' node ids unchanged. --- CLAUDE.md | 24 ++++ .../backend/impl/postgres/async_support.py | 116 ++++++++++++++++++ .../backend/impl/postgres/cli/introspect.py | 4 +- .../backend/impl/postgres/cli/query.py | 4 +- .../backend/impl/postgres/cli/status.py | 4 +- tests/conftest.py | 32 +++-- .../feature/backend/conftest.py | 10 -- .../feature/backend/test_async_support.py | 115 +++++++++++++++++ .../feature/connection/conftest.py | 12 -- 9 files changed, 285 insertions(+), 36 deletions(-) create mode 100644 src/rhosocial/activerecord/backend/impl/postgres/async_support.py create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/test_async_support.py diff --git a/CLAUDE.md b/CLAUDE.md index 45533eeb..569a6bef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,6 +35,30 @@ This project is under active development. Key features implemented: This project adheres to the same version control, branching, commit message, and changelog management standards as the main `python-activerecord` project. +## Async Backend and the Windows Event Loop + +psycopg's async mode waits on the libpq socket with `loop.add_reader` / +`loop.add_writer`, which on Windows the platform default event loop +(`ProactorEventLoop`, since 3.8) does not implement. Therefore: + +- The sync path is unaffected everywhere. +- `AsyncPostgresBackend` requires a selector-based event loop on Windows + (`SelectorEventLoop(SelectSelector())`). psycopg 3.3+ detects the + mismatch at connect time and raises with the fix named; older versions + surface asyncio's raw `NotImplementedError`. +- To run async code on Windows, use the helper this backend ships: + `run_coroutine(coro)` from + `rhosocial.activerecord.backend.impl.postgres.async_support` -- it + constructs the required loop and gives it `asyncio.run`-grade + lifecycle (cancel on `KeyboardInterrupt`, shutdown of async generators, + close, detach). It may only be called when no loop is running. +- The CLI's `--async` modes (`introspect`, `query`, `status`) already use + it, so they work on Windows out of the box. +- Do NOT "fix" this by switching the global event loop policy: + `WindowsSelectorEventLoopPolicy` is deprecated for removal in Python + 3.16, and a selector loop on Windows cannot serve asyncio subprocess or + pipe I/O, which would break other parts of the process. + ## Search Scope - Use `rg` for repository searches; never use `grep`. diff --git a/src/rhosocial/activerecord/backend/impl/postgres/async_support.py b/src/rhosocial/activerecord/backend/impl/postgres/async_support.py new file mode 100644 index 00000000..1232f21b --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/async_support.py @@ -0,0 +1,116 @@ +# src/rhosocial/activerecord/backend/impl/postgres/async_support.py +"""Running psycopg's async mode on Windows, and one way to run it everywhere. + +psycopg waits on the libpq socket with ``loop.add_reader`` / +``loop.add_writer`` (see ``psycopg/waiting.py``), and those two methods are +the selector-based event loop's -- on Windows the platform default since +3.8 is the ProactorEventLoop, which does not implement them. So every +"await psycopg async under the default loop" path fails there, including +this backend's own CLI in its ``--async`` modes. + +psycopg 3.3+ detects the mismatch at connect time and raises with the fix +spelled out; older versions surface asyncio's raw ``NotImplementedError``. +This module applies the fix once, for the CLI and for callers who would +rather not memorize it: construct the loop psycopg's async needs, and run +coroutines on it. + +Why the loop is constructed rather than asked of the policy: the only +supported way to obtain one on Windows is to build it, and +``asyncio.run(..., loop_factory=...)`` is 3.13+, while this backend still +supports 3.8. One explicit construction serves every supported version. +Why no policy switch: swapping the platform policy is a global side effect, +it would take asyncio subprocess and pipe I/O down with it, and the policy +class is deprecated for removal in 3.16 besides. + +Not needed by the sync path at all, and not needed by aiosqlite either -- +that driver blocks in a thread rather than registering the socket with the +loop. +""" + +import asyncio +import selectors +from typing import Awaitable, TypeVar + +__all__ = ["run_coroutine", "selector_event_loop"] + +_T = TypeVar("_T") + + +def selector_event_loop() -> asyncio.AbstractEventLoop: + """The event loop psycopg's async mode requires. + + Constructed on every platform: on Windows this is the fix itself (the + ProactorEventLoop has no ``add_reader``/``add_writer``), elsewhere it is + the same family of loop the platform already defaults to, so one code + path behaves identically everywhere. + + Returns: + A ``SelectorEventLoop`` bound to a ``SelectSelector``. What matters + about it is not its class name but the two methods psycopg calls on + it -- ``add_reader`` and ``add_writer`` -- which this loop has. + """ + return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +def run_coroutine(coro: Awaitable[_T]) -> _T: + """Run *coro* to completion on the loop psycopg's async mode requires. + + The lifecycle mirrors ``asyncio.run``: the loop is installed as the + current one for the duration, a normal or exceptional return still + shuts async generators down and closes the loop, and ``KeyboardInterrupt`` + cancels what is still pending before the loop goes away, so a Ctrl-C in + a CLI leaves no task attached to a loop that no longer exists. + + May only be called when no loop is running -- exactly the precondition + ``asyncio.run`` states, and enforced the same way (a ``RuntimeError`` + from the loop underneath). + + Args: + coro: The coroutine (or awaitable) to run to completion. + + Returns: + Whatever *coro* returns. + + Raises: + RuntimeError: If called while an event loop is already running. + KeyboardInterrupt: Propagated after the pending tasks are cancelled. + """ + loop = selector_event_loop() + try: + asyncio.set_event_loop(loop) + return loop.run_until_complete(coro) + except KeyboardInterrupt: + _cancel_pending(loop) + raise + finally: + _shutdown(loop) + + +def _cancel_pending(loop: asyncio.AbstractEventLoop) -> None: + """Cancel what the interrupted run left pending, and drain it. + + The same shape as asyncio's own interrupt handling: cancellations are + delivered and awaited once, so nothing outlives the loop. + """ + try: + pending = asyncio.all_tasks(loop) + except RuntimeError: # no running loop: nothing is pending + return + for task in pending: + task.cancel() + if pending: + loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + + +def _shutdown(loop: asyncio.AbstractEventLoop) -> None: + """Close *loop* and detach it as the current one, swallowing shutdown noise. + + An async generator that fails to shut down must not replace the + coroutine's own outcome; asyncio.run makes the same trade. + """ + try: + loop.run_until_complete(loop.shutdown_asyncgens()) + except BaseException: # noqa: BLE001 - shutdown must not mask the result + pass + asyncio.set_event_loop(None) + loop.close() diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py index 4d72434a..784dd5ad 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py @@ -11,12 +11,12 @@ """ import argparse -import asyncio import sys from dataclasses import asdict, is_dataclass from enum import Enum from typing import Any, Dict, List +from rhosocial.activerecord.backend.impl.postgres.async_support import run_coroutine from rhosocial.activerecord.backend.impl.postgres.backend.async_backend import AsyncPostgresBackend from rhosocial.activerecord.backend.impl.postgres.backend import PostgresBackend from rhosocial.activerecord.backend.errors import ConnectionError, QueryError @@ -208,7 +208,7 @@ def handle(args): if args.is_async: backend = AsyncPostgresBackend(connection_config=config) - asyncio.run(_handle_introspect_async(args, backend, provider)) + run_coroutine(_handle_introspect_async(args, backend, provider)) else: backend = PostgresBackend(connection_config=config) _handle_introspect_sync(args, backend, provider) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py index a6efb32a..ff637b79 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py @@ -5,10 +5,10 @@ """ import argparse -import asyncio import logging import sys +from rhosocial.activerecord.backend.impl.postgres.async_support import run_coroutine from rhosocial.activerecord.backend.impl.postgres.backend.async_backend import AsyncPostgresBackend from rhosocial.activerecord.backend.impl.postgres.backend import PostgresBackend from rhosocial.activerecord.backend.errors import ConnectionError, QueryError @@ -142,7 +142,7 @@ def handle(args): if args.is_async: backend = AsyncPostgresBackend(connection_config=config) - asyncio.run(_execute_query_async(sql_source, backend, provider, **kwargs)) + run_coroutine(_execute_query_async(sql_source, backend, provider, **kwargs)) else: backend = PostgresBackend(connection_config=config) _execute_query_sync(sql_source, backend, provider, **kwargs) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py index a7a938a8..17769175 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py @@ -6,13 +6,13 @@ """ import argparse -import asyncio import json import sys from dataclasses import asdict, is_dataclass from enum import Enum from typing import Any +from rhosocial.activerecord.backend.impl.postgres.async_support import run_coroutine from rhosocial.activerecord.backend.impl.postgres.backend.async_backend import AsyncPostgresBackend from rhosocial.activerecord.backend.impl.postgres.backend import PostgresBackend from rhosocial.activerecord.backend.errors import ConnectionError, QueryError @@ -113,7 +113,7 @@ def handle(args): if args.is_async: backend = AsyncPostgresBackend(connection_config=config) - asyncio.run(_handle_status_async(args, backend, provider)) + run_coroutine(_handle_status_async(args, backend, provider)) else: backend = PostgresBackend(connection_config=config) _handle_status_sync(args, backend, provider) diff --git a/tests/conftest.py b/tests/conftest.py index 4d68a27f..cfd3b1c8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,19 +8,35 @@ """ import asyncio import os +import selectors import sys import pytest -def ensure_compatible_event_loop(): - if sys.platform == "win32": - policy = asyncio.get_event_loop_policy() - if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - - -ensure_compatible_event_loop() +if sys.platform == "win32": + + def pytest_asyncio_loop_factories(config, item): + """Give async tests the event loop psycopg's async mode requires. + + On Windows the platform default is the ProactorEventLoop, which does + not implement the ``add_reader``/``add_writer`` methods psycopg + waits on (psycopg/waiting.py), so every async backend test would + fail at connect. A selector loop is what the driver can use, and + this hook is how pytest-asyncio (1.x) is told to build one -- the + supported replacement for the deprecated + ``WindowsSelectorEventLoopPolicy``, which also had the drawback of + switching the loop for the whole process. + + The hook exists only on Windows, and pytest-asyncio requires a + non-empty mapping from it -- so it cannot answer "nothing" on other + platforms to keep them untouched; not defining it is how that is + said. Elsewhere the default loop is already selector-based, and the + async tests' node ids stay exactly as they were. + """ + return { + "selector": lambda: asyncio.SelectorEventLoop(selectors.SelectSelector()) + } # Set the environment variable that the testsuite uses to locate the provider registry. diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py index 3fdec40c..5aa30bc9 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py @@ -1,6 +1,4 @@ # tests/rhosocial/activerecord_postgres_test/feature/backend/conftest.py -import asyncio -import sys import pytest import pytest_asyncio import yaml @@ -12,14 +10,6 @@ from rhosocial.activerecord.backend.impl.postgres.config import PostgresConnectionConfig -def ensure_compatible_event_loop(): - if sys.platform == "win32": - policy = asyncio.get_event_loop_policy() - if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - -ensure_compatible_event_loop() - # --- Scenario Loading Logic --- SCENARIO_MAP: Dict[str, Dict[str, Any]] = {} diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_async_support.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_async_support.py new file mode 100644 index 00000000..73c8ddd1 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_async_support.py @@ -0,0 +1,115 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_async_support.py +"""The loop psycopg's async mode needs, and the runner that provides it. + +No database is involved: what is pinned here is the contract the driver +depends on. psycopg waits on the libpq socket through +``loop.add_reader``/``loop.add_writer`` (psycopg/waiting.py), and on Windows +the platform default loop does not implement them -- which is why the +helper builds the loop explicitly instead of asking the policy for one. + +The capability assertions matter more than the class-name ones: if CPython +ever ships a different loop that keeps the methods, these tests should stay +green, and if a loop loses them, these tests are what says so first. +""" + +import asyncio + +import pytest + +from rhosocial.activerecord.backend.impl.postgres.async_support import ( + run_coroutine, + selector_event_loop, +) + + +class TestTheLoopTheDriverNeeds: + def test_the_loop_implements_what_psycopg_calls_on_it(self): + loop = selector_event_loop() + try: + assert callable(loop.add_reader) + assert callable(loop.add_writer) + finally: + loop.close() + + def test_the_loop_is_selector_based_where_that_is_the_fix(self): + """On Windows the selector loop is the fix itself; elsewhere it is + the same family the platform already defaults to, so one code path + behaves identically everywhere.""" + loop = selector_event_loop() + try: + assert isinstance(loop, asyncio.SelectorEventLoop) + assert type(loop).__module__.startswith("asyncio") + finally: + loop.close() + + +class TestRunCoroutine: + def test_runs_the_coroutine_and_returns_its_value(self): + async def answer(): + await asyncio.sleep(0) + return 42 + + assert run_coroutine(answer()) == 42 + + def test_propagates_the_coroutine_s_exception(self): + async def boom(): + await asyncio.sleep(0) + raise ValueError("from the coroutine") + + with pytest.raises(ValueError, match="from the coroutine"): + run_coroutine(boom()) + + def test_the_loop_the_coroutine_ran_on_is_closed_afterwards(self): + """Nothing is left running: the loop the coroutine used is closed by + the time run_coroutine returns.""" + captured = {} + + async def grab(): + captured["loop"] = asyncio.get_running_loop() + return "done" + + assert run_coroutine(grab()) == "done" + assert captured["loop"].is_closed() + + def test_refuses_to_run_inside_a_running_loop(self): + """Same precondition asyncio.run states: a runner is for the top of + an async stack, not for nesting one loop inside another.""" + + async def nested(): + coro = _trivial() + try: + return run_coroutine(coro) + finally: + coro.close() + + async def _trivial(): + return 1 + + with pytest.raises(RuntimeError): + asyncio.run(nested()) + + def test_interrupt_cancels_what_is_pending_before_the_loop_closes(self): + """Ctrl-C in a CLI must not leave tasks on a loop that then goes + away: the pending ones are cancelled and drained. + + The interrupt is raised from a scheduled callback, which is how a + real one arrives -- it propagates out of the running loop rather + than surfacing as an ordinary exception inside the coroutine. + """ + cancelled = [] + + def interrupt(): + raise KeyboardInterrupt + + async def long_lived(): + loop = asyncio.get_running_loop() + loop.call_soon(interrupt) + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + cancelled.append("cancelled") + raise + + with pytest.raises(KeyboardInterrupt): + run_coroutine(long_lived()) + assert cancelled == ["cancelled"] diff --git a/tests/rhosocial/activerecord_postgres_test/feature/connection/conftest.py b/tests/rhosocial/activerecord_postgres_test/feature/connection/conftest.py index 91197815..540b149e 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/connection/conftest.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/connection/conftest.py @@ -5,8 +5,6 @@ This module provides fixtures for testing connection pools with PostgreSQL backends. """ -import asyncio -import sys import os from typing import Dict, Any, Generator @@ -24,16 +22,6 @@ ) -def ensure_compatible_event_loop(): - if sys.platform == "win32": - policy = asyncio.get_event_loop_policy() - if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): - new_policy = asyncio.WindowsSelectorEventLoopPolicy() - asyncio.set_event_loop_policy(new_policy) - - -ensure_compatible_event_loop() - # --- Scenario Loading Logic --- SCENARIO_MAP: Dict[str, Dict[str, Any]] = {} From 4cbacaba214ec64021fe6ad0090b779d80f424e4 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 1 Nov 2026 17:32:08 +0800 Subject: [PATCH 03/27] refactor(introspection)!: name the PostgreSQL introspectors after the family, not after the sync Follows the core rename (53842f99) that took the `Sync` prefix off the introspection family's sync side, so the pair's shape now matches the backend family's: `PostgreSQLIntrospector` and `AsyncPostgreSQLIntrospector`, one plain name and one marked variant, the way `PostgresBackend` and `AsyncPostgresBackend` already read. Both concrete classes drop the prefix -- `SyncPostgreSQLIntrospector` and `SyncPostgreSQLStatusIntrospector` -- along with the three core base names they and their imports refer to (`SyncAbstractIntrospector`, `SyncAbstractStatusIntrospector`, `SyncIntrospectorExecutor`), which the core change renamed underneath us. Constructor call sites in the backend and sync backend, the package's exports, the introspection and dialect tests and the status tests follow. Alias-free by the same reasoning as the core change: a `SyncXxx = Xxx` abbreviation would put two names on one fact, and no other family here spells a pair that way. Method names and the sync/async parity they carry are untouched -- this commit only changes what the classes are called. Verified: imports of all four classes through the package's own export surface, black showing no new debt in any touched file (the dirt in these files exists at HEAD and renaming only ever shortens a line), ruff clean, and a pytest collection pass over the four touched test modules (11/74/29/15 cases) with no NameError or ImportError. The suite needs a live PostgreSQL server to run further, which CI provides. --- .../backend/impl/postgres/backend/backend.py | 8 +++---- .../backend/impl/postgres/backend/sync.py | 8 +++---- .../impl/postgres/introspection/__init__.py | 8 +++---- .../postgres/introspection/introspector.py | 22 +++++++++---------- .../introspection/status_introspector.py | 10 ++++----- .../dialect/test_introspection_parsing.py | 6 ++--- .../test_materialized_view_introspection.py | 6 ++--- .../introspection/test_introspection_basic.py | 6 ++--- .../test_introspection_default_schema.py | 6 ++--- .../introspection/test_status_introspector.py | 4 ++-- 10 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/backend/backend.py b/src/rhosocial/activerecord/backend/impl/postgres/backend/backend.py index 5217485d..60c1a795 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/backend/backend.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/backend/backend.py @@ -45,7 +45,7 @@ from .base import PostgresBackendMixin, PostgresConcurrencyMixin from ..protocols import PostgresExtensionInfo from ..transaction import PostgresTransactionManager -from ..introspection import SyncPostgreSQLIntrospector +from ..introspection import PostgreSQLIntrospector if TYPE_CHECKING: from ..expression import PostgresCopyFromStatement, PostgresCopyToStatement @@ -165,10 +165,10 @@ def __init__(self, **kwargs: Any) -> None: self.log(logging.INFO, "PostgresBackend initialized") def _create_introspector(self) -> Any: - """Create and return a SyncPostgreSQLIntrospector with a sync executor.""" - from rhosocial.activerecord.backend.introspection.executor import SyncIntrospectorExecutor + """Create and return a PostgreSQLIntrospector with a sync executor.""" + from rhosocial.activerecord.backend.introspection.executor import IntrospectorExecutor - return SyncPostgreSQLIntrospector(self, SyncIntrospectorExecutor(self)) + return PostgreSQLIntrospector(self, IntrospectorExecutor(self)) @property def dialect(self) -> PostgresDialect: diff --git a/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py b/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py index 7376a90f..7ab91246 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/backend/sync.py @@ -45,7 +45,7 @@ from .base import PostgresBackendMixin, PostgresConcurrencyMixin from ..protocols import PostgresExtensionInfo from ..transaction import PostgresTransactionManager -from ..introspection import SyncPostgreSQLIntrospector +from ..introspection import PostgreSQLIntrospector if TYPE_CHECKING: from ..expression import PostgresCopyFromStatement, PostgresCopyToStatement @@ -165,10 +165,10 @@ def __init__(self, **kwargs: Any) -> None: self.log(logging.INFO, "PostgresBackend initialized") def _create_introspector(self) -> Any: - """Create and return a SyncPostgreSQLIntrospector with a sync executor.""" - from rhosocial.activerecord.backend.introspection.executor import SyncIntrospectorExecutor + """Create and return a PostgreSQLIntrospector with a sync executor.""" + from rhosocial.activerecord.backend.introspection.executor import IntrospectorExecutor - return SyncPostgreSQLIntrospector(self, SyncIntrospectorExecutor(self)) + return PostgreSQLIntrospector(self, IntrospectorExecutor(self)) @property def dialect(self) -> PostgresDialect: diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/__init__.py index e9b81692..258360fc 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/__init__.py @@ -1,10 +1,10 @@ # src/rhosocial/activerecord/backend/impl/postgres/introspection/__init__.py from .introspector import ( - SyncPostgreSQLIntrospector, + PostgreSQLIntrospector, AsyncPostgreSQLIntrospector, ) from .status_introspector import ( - SyncPostgreSQLStatusIntrospector, + PostgreSQLStatusIntrospector, AsyncPostgreSQLStatusIntrospector, ) from .status_types import ( @@ -16,9 +16,9 @@ ) __all__ = [ - "SyncPostgreSQLIntrospector", + "PostgreSQLIntrospector", "AsyncPostgreSQLIntrospector", - "SyncPostgreSQLStatusIntrospector", + "PostgreSQLStatusIntrospector", "AsyncPostgreSQLStatusIntrospector", # PostgreSQL's own status families, taken back from core "WALInfo", diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py index 634acd68..350a4218 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py @@ -2,7 +2,7 @@ """ PostgreSQL concrete introspectors. -Implements SyncAbstractIntrospector and AsyncAbstractIntrospector for PostgreSQL +Implements AbstractIntrospector and AsyncAbstractIntrospector for PostgreSQL databases using pg_catalog system tables for metadata queries. The introspectors are exposed via ``backend.introspector``. @@ -20,7 +20,7 @@ - Default schema is 'public' Design principle: Sync and Async are separate and cannot coexist. -- SyncPostgreSQLIntrospector: for synchronous backends +- PostgreSQLIntrospector: for synchronous backends - AsyncPostgreSQLIntrospector: for asynchronous backends """ @@ -28,12 +28,12 @@ from rhosocial.activerecord.backend.introspection.base import ( IntrospectorMixin, - SyncAbstractIntrospector, + AbstractIntrospector, AsyncAbstractIntrospector, ) from rhosocial.activerecord.backend.introspection.cache import IntrospectionCache from rhosocial.activerecord.backend.introspection.executor import ( - SyncIntrospectorExecutor, + IntrospectorExecutor, AsyncIntrospectorExecutor, ) from rhosocial.activerecord.backend.introspection.status import ExtensionInfo @@ -54,7 +54,7 @@ ) from rhosocial.activerecord.backend.expression.types import DataType from .status_introspector import ( - SyncPostgreSQLStatusIntrospector, + PostgreSQLStatusIntrospector, AsyncPostgreSQLStatusIntrospector, ) from .types import SequenceInfo @@ -72,7 +72,7 @@ class PostgreSQLIntrospectorMixin(IntrospectorMixin): """Mixin providing shared PostgreSQL-specific introspection logic. - Both SyncPostgreSQLIntrospector and AsyncPostgreSQLIntrospector inherit + Both PostgreSQLIntrospector and AsyncPostgreSQLIntrospector inherit from this mixin to share: - Default schema handling - parse_* implementations @@ -621,7 +621,7 @@ def _decode_trigger_events(tgtype: int) -> List[str]: return events if events else ["UPDATE"] -class SyncPostgreSQLIntrospector(PostgreSQLIntrospectorMixin, SyncAbstractIntrospector): +class PostgreSQLIntrospector(PostgreSQLIntrospectorMixin, AbstractIntrospector): """Synchronous introspector for PostgreSQL backends. Uses pg_catalog system tables rather than information_schema for @@ -640,17 +640,17 @@ class SyncPostgreSQLIntrospector(PostgreSQLIntrospectorMixin, SyncAbstractIntros def __init__( self, backend: Any, - executor: SyncIntrospectorExecutor, + executor: IntrospectorExecutor, cache: Optional[IntrospectionCache] = None, ) -> None: super().__init__(backend, executor, cache) - self._status_instance: Optional[SyncPostgreSQLStatusIntrospector] = None + self._status_instance: Optional[PostgreSQLStatusIntrospector] = None @property - def status(self) -> SyncPostgreSQLStatusIntrospector: + def status(self) -> PostgreSQLStatusIntrospector: """PostgreSQL status introspector (lazily created).""" if self._status_instance is None: - self._status_instance = SyncPostgreSQLStatusIntrospector(self._backend) + self._status_instance = PostgreSQLStatusIntrospector(self._backend) return self._status_instance def list_materialized_views( diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_introspector.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_introspector.py index 95136781..488104b2 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_introspector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/status_introspector.py @@ -6,7 +6,7 @@ pg_stat_activity, and other system views. Design principle: Sync and Async are separate and cannot coexist. -- SyncPostgreSQLStatusIntrospector: for synchronous backends +- PostgreSQLStatusIntrospector: for synchronous backends - AsyncPostgreSQLStatusIntrospector: for asynchronous backends """ @@ -21,7 +21,7 @@ StorageInfo, SessionInfo, ExtensionInfo, - SyncAbstractStatusIntrospector, + AbstractStatusIntrospector, AsyncAbstractStatusIntrospector, ) from ..expression.status import ( @@ -118,7 +118,7 @@ class PostgreSQLStatusIntrospectorMixin: """Mixin providing shared PostgreSQL status introspection logic. - Both SyncPostgreSQLStatusIntrospector and AsyncPostgreSQLStatusIntrospector + Both PostgreSQLStatusIntrospector and AsyncPostgreSQLStatusIntrospector inherit from this mixin to share: - Query execution helpers - parse_* implementations @@ -272,8 +272,8 @@ def _build_server_overview( ) -class SyncPostgreSQLStatusIntrospector( - PostgreSQLStatusIntrospectorMixin, SyncAbstractStatusIntrospector +class PostgreSQLStatusIntrospector( + PostgreSQLStatusIntrospectorMixin, AbstractStatusIntrospector ): """Synchronous PostgreSQL status introspector. diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_introspection_parsing.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_introspection_parsing.py index 9b1225d7..190019f9 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_introspection_parsing.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_introspection_parsing.py @@ -28,7 +28,7 @@ from rhosocial.activerecord.backend.impl.postgres.introspection.introspector import ( AsyncPostgreSQLIntrospector, PostgreSQLIntrospectorMixin, - SyncPostgreSQLIntrospector, + PostgreSQLIntrospector, ) from rhosocial.activerecord.backend.introspection.status import ExtensionInfo from rhosocial.activerecord.backend.introspection.types import TableType @@ -224,11 +224,11 @@ def test_sync_and_async_read_the_same_row(self): # The statement is the dialect's and the parser is the mixin's, so # neither path can drift into reading a different shape. assert ( - SyncPostgreSQLIntrospector.parse_database_info + PostgreSQLIntrospector.parse_database_info is AsyncPostgreSQLIntrospector.parse_database_info ) assert ( - SyncPostgreSQLIntrospector.parse_database_info + PostgreSQLIntrospector.parse_database_info is PostgreSQLIntrospectorMixin.parse_database_info ) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_materialized_view_introspection.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_materialized_view_introspection.py index 28eb5b76..7e16c977 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_materialized_view_introspection.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/dialect/test_materialized_view_introspection.py @@ -22,7 +22,7 @@ from rhosocial.activerecord.backend.impl.postgres.introspection.introspector import ( AsyncPostgreSQLIntrospector, PostgreSQLIntrospectorMixin, - SyncPostgreSQLIntrospector, + PostgreSQLIntrospector, ) @@ -141,12 +141,12 @@ class TestMaterializedViewIntrospectionSurface: "method", ["list_materialized_views", "get_materialized_view_info", "materialized_view_exists"] ) def test_sync_and_async_expose_same_methods(self, method): - assert hasattr(SyncPostgreSQLIntrospector, method) + assert hasattr(PostgreSQLIntrospector, method) assert hasattr(AsyncPostgreSQLIntrospector, method) def test_cache_keys_are_namespaced(self): """MV lookups must not collide with regular view lookups.""" - introspector = SyncPostgreSQLIntrospector.__new__(SyncPostgreSQLIntrospector) + introspector = PostgreSQLIntrospector.__new__(PostgreSQLIntrospector) introspector._cache = InMemoryIntrospectionCache() list_key = introspector._materialized_view_cache_key("public", include_system=False) single_key = introspector._materialized_view_cache_key( diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_basic.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_basic.py index 3a967183..7f2ea667 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_basic.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_basic.py @@ -11,7 +11,7 @@ from rhosocial.activerecord.backend.introspection.types import IntrospectionDatabaseInfo from rhosocial.activerecord.backend.introspection.backend_mixin import IntrospectorBackendMixin -from rhosocial.activerecord.backend.introspection.base import SyncAbstractIntrospector +from rhosocial.activerecord.backend.introspection.base import AbstractIntrospector class TestIntrospectionSupport: @@ -23,8 +23,8 @@ def test_has_introspector(self, postgres_backend_single): assert postgres_backend_single.introspector is not None def test_introspector_is_abstract_introspector(self, postgres_backend_single): - """Test that introspector is a SyncAbstractIntrospector instance.""" - assert isinstance(postgres_backend_single.introspector, SyncAbstractIntrospector) + """Test that introspector is a AbstractIntrospector instance.""" + assert isinstance(postgres_backend_single.introspector, AbstractIntrospector) def test_backend_is_introspector_mixin(self, postgres_backend_single): """Test that backend uses IntrospectorBackendMixin.""" diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_default_schema.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_default_schema.py index aea549bc..922e8a99 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_default_schema.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_default_schema.py @@ -14,7 +14,7 @@ from rhosocial.activerecord.backend.impl.postgres.config import PostgresConnectionConfig from rhosocial.activerecord.backend.impl.postgres.dialect import PostgresDialect from rhosocial.activerecord.backend.impl.postgres.introspection import ( - SyncPostgreSQLIntrospector, + PostgreSQLIntrospector, ) @@ -30,9 +30,9 @@ def make_config(**overrides) -> PostgresConnectionConfig: return PostgresConnectionConfig(**params) -def make_introspector(config: PostgresConnectionConfig) -> SyncPostgreSQLIntrospector: +def make_introspector(config: PostgresConnectionConfig) -> PostgreSQLIntrospector: backend = PostgresBackend(connection_config=config) - return SyncPostgreSQLIntrospector(backend, executor=object()) + return PostgreSQLIntrospector(backend, executor=object()) class TestIntrospectorDefaultSchema: diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_status_introspector.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_status_introspector.py index 1d05a758..a3da7481 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_status_introspector.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_status_introspector.py @@ -2,7 +2,7 @@ """ Tests for PostgreSQL status introspector. -This module tests the SyncPostgreSQLStatusIntrospector functionality +This module tests the PostgreSQLStatusIntrospector functionality for retrieving server status information via pg_settings, pg_stat_activity, and other system views. """ @@ -28,7 +28,7 @@ ) -class TestSyncPostgreSQLStatusIntrospector: +class TestPostgreSQLStatusIntrospector: """Tests for synchronous PostgreSQL status introspector.""" def test_get_overview(self, postgres_backend_single): From 14508505a2073989d75c13167ad4ff864f05a883 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Mon, 2 Nov 2026 15:29:14 +0800 Subject: [PATCH 04/27] docs(introspection): say what supports_extensions promises, and stop over-promising it `extension` is not SQL vocabulary, and the object it names differs per engine: PostgreSQL's are `CREATE EXTENSION` packages carrying a version and a schema, SQLite's are registered virtual table modules. The switch was proposed for withdrawal from core on exactly those grounds, and that argument loses. Withdrawing it would not make the semantics agree -- it would only make the question unaskable through one spelling, so a caller would have to already know the backend's private name (`supports_extension_introspection`) before it could ask anything. That is strictly worse than an answer that has to be read together with its backend's docs. The standing precedent is `supports_fulltext_search`: `MATCH ... AGAINST` and `CONTAINS` share nothing but the question, and the name stays in core because a caller wants one way to ask it. So the rule is now written down -- a capability switch buys *discoverability*, never *semantics* -- as section 2a of the introspection rules, with the test for deciding which of the two is uniform, and the consequences for anyone editing such a probe. Two things the rule caught while being written: * SQLite answered `True` on the strength of `PRAGMA module_list` alone, with no query formatting, no result type and no CLI type behind it. A caller was promised an answer it could not get, which is worse than `False` -- `False` is "I do not answer", whereas that is "I will" followed by nothing. It now answers `False`, with the honest name for the thing it would need to build. * PostgreSQL answered `True` through its own spelling while the core spelling still defaulted to `False`: one fact, two answers, so a caller could be told "unsupported" by whichever name it happened to use. It now answers through the core name too, and the CLI keeps working because it never used it. Verified: SQLite `supports_extensions` is False, PostgreSQL answers True under both spellings, ruff clean, and no new black debt in the touched files (the dirt in them is present at HEAD). --- .../backend/impl/postgres/mixins/extension.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/extension.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/extension.py index e886ea3f..87c146fc 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/extension.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/extension.py @@ -26,9 +26,24 @@ def supports_extension_introspection(self) -> bool: The capability for the ``introspect extensions`` family. Declared beside the extension detection it describes, so a caller can ask which extensions are installed rather than assuming. + + This is PostgreSQL's own spelling, not the core one: ``extension`` + means a ``CREATE EXTENSION`` package here, and no engine's flavor of + the concept is interchangeable with it. """ return True + def supports_extensions(self) -> bool: + """The same capability under the core name. + + ``True`` so a consumer asking the shared protocol gets the answer, + instead of the split this closes: ``supports_extensions`` (core) said + ``False`` by default while this dialect's own + ``supports_extension_introspection`` said ``True``, and a caller + could be told "unsupported" by whichever name it happened to use. + """ + return self.supports_extension_introspection() + KNOWN_EXTENSIONS = { # Spatial/Geospatial "postgis": { From 5d25ae9441ba69a3648a271d62c52183ab625093 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Mon, 2 Nov 2026 17:10:33 +0800 Subject: [PATCH 05/27] feat(introspection): PostgreSQL answers per-table usage counters First backend answer for `supports_table_stats`, from the source the family was shaped on: `pg_stat_user_tables`. The projection is the collector's own columns under their own names -- `seq_scan`, `idx_scan`, `n_tup_ins`, `n_tup_upd`, `n_tup_del`, `n_live_tup` -- which is why the common shape reads them by those spellings first. What the listing deliberately does not do is flatten the view's other facts into the common shape: `n_dead_tup` and the `last_*` access times are real and useful, but they have no name every engine shares, so projecting them into the common slots would be the `extra`-as-parking-spot move the ownership rule forbids in the other direction. They stay on `pg_stat_user_tables`, for a backend reader that wants them. A table with no statistics row yet is absent from the view rather than zero-filled, and the docstring says so: the listing answers for the tables the collector has seen, which is the honest scope. The reset caveat -- crash or `pg_stat_reset()`, so every counter is "since the last reset" -- is stated on the capability probe, where a caller deciding what a zero means will look. --- .../impl/postgres/mixins/introspection.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index baed94ab..01cc7fe7 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -39,6 +39,7 @@ ViewInfoQueryStatement, TriggerListQueryStatement, TriggerInfoQueryStatement, + TableUsageStatsQueryStatement, ) from ..expression.introspection import ( PostgresMaterializedViewInfoQueryStatement, @@ -142,6 +143,19 @@ def supports_table_storage_introspection(self) -> bool: """ return True + def supports_table_stats(self) -> bool: + """PostgreSQL answers per-table usage counters from pg_stat_user_tables. + + The counters are the statistics collector's, reset on crash or by + ``pg_stat_reset()`` -- so "since the last reset" is the only tense the + numbers have, and a zero is "no access in the window". The view also + carries facts with no common name (dead tuples, last-access times); + those are real, but they are not projected here, and where they go is + the backend reader's decision, not the common shape's. + """ + return True + + def get_supported_introspection_scopes(self) -> List[IntrospectionScope]: """Get list of supported introspection scopes.""" return [ @@ -2632,4 +2646,52 @@ def format_postgres_enum_type_values_query( ) + def format_table_usage_stats_query_statement( + self, expr: "TableUsageStatsQueryStatement" + ) -> Tuple[str, tuple]: + """Format the per-table usage-counter listing from pg_stat_user_tables. + + One row per table the collector has seen in the schema, projected from + its own columns: ``seq_scan`` / ``idx_scan`` for the reads, + ``n_tup_ins`` / ``n_tup_upd`` / ``n_tup_del`` for the writes, + ``n_live_tup`` for the row estimate the counters were measured + against. The view's other facts -- ``n_dead_tup``, the ``last_*`` + access times -- are real and useful but have no common name, so they + are not projected: this listing is the common shape, and the engine's + extras belong on a backend reader beside it. + + A table with no statistics row yet is simply absent from + ``pg_stat_user_tables``; the listing answers for the tables the + collector has seen, which is the honest scope rather than a + zero-filled one. + """ + from rhosocial.activerecord.backend.expression import ( + ComparisonPredicate, + ) + + params = expr.get_params() + schema = params.get("schema") or self.default_schema + + source = self._catalog_relation("pg_stat_user_tables", "s") + + conditions = [ + ComparisonPredicate( + self, "eq", self._catalog_column("schemaname", "s"), self._catalog_value(schema) + ), + ] + projections = [ + self._catalog_column("schemaname", "s"), + self._catalog_column("relname", "s"), + self._catalog_column("seq_scan", "s"), + self._catalog_column("idx_scan", "s"), + self._catalog_column("n_tup_ins", "s"), + self._catalog_column("n_tup_upd", "s"), + self._catalog_column("n_tup_del", "s"), + self._catalog_column("n_live_tup", "s"), + ] + return self._catalog_select( + projections, source, conditions, [self._catalog_column("relname", "s")] + ) + + __all__ = ["PostgresIntrospectionCapabilityMixin"] From d7a141dd6de80084cf4f5842f6ce1082162c81e7 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Mon, 2 Nov 2026 18:36:52 +0800 Subject: [PATCH 06/27] fix(introspection): PostgreSQL renders the unused-indexes listing instead of falling through The roundtrip walk in the PostgreSQL suite enumerates every core expression class and asks this dialect to render it, classifying each answer as a render, a declared `UnsupportedFeatureError`, or an entry in `LEGITIMATE_NON_RENDERS`. The new `UnusedIndexesQueryStatement` had none of those: it reached a `runtime_checkable` protocol stub sitting in the dialect's MRO, which answers `getattr` and then returns `None` -- reported as `ProtocolNotImplementedError`, a defect every job flagged. So PostgreSQL answers it properly rather than being excused from it: no vendor view exists, so the family is computed from `pg_stat_user_indexes.idx_scan` being zero over valid indexes, joined to `pg_namespace` for the schema. Only `idx_scan` is read -- an index is what makes a scan indexed -- and `indisvalid` filters the indexes the planner would never choose. The capability docstring says which tense PostgreSQL speaks, because it is not MySQL's: a zero means "no index scan since the last crash or `pg_stat_reset()`", an engine-computed claim rather than a vendor-maintained one, and the answer stays "worth reviewing" rather than "safe to drop". Rendered locally for the record: `SELECT n.nspname AS "schema_name", i.relname AS "table_name", i.indexrelname AS "index_name" FROM pg_stat_user_indexes AS i JOIN pg_namespace AS n ON i.schemaname = n.nspname WHERE n.nspname = %s AND i.idx_scan = %s AND i.indisvalid = TRUE ORDER BY i.relname, i.indexrelname`. --- .../impl/postgres/mixins/introspection.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 01cc7fe7..9fc309ce 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -40,6 +40,7 @@ TriggerListQueryStatement, TriggerInfoQueryStatement, TableUsageStatsQueryStatement, + UnusedIndexesQueryStatement, ) from ..expression.introspection import ( PostgresMaterializedViewInfoQueryStatement, @@ -156,6 +157,18 @@ def supports_table_stats(self) -> bool: return True + def supports_unused_indexes_detection(self) -> bool: + """PostgreSQL answers from ``pg_stat_user_indexes.idx_scan``. + + A zero counter means "no index scan since the statistics were last + reset", which is PostgreSQL's tense for this family and a weaker + claim than MySQL's ``sys.schema_unused_indexes``: an index untouched + since a crash, or since a ``pg_stat_reset()``, reports zero. The + answer is "worth reviewing", never "safe to drop". + """ + return True + + def get_supported_introspection_scopes(self) -> List[IntrospectionScope]: """Get list of supported introspection scopes.""" return [ @@ -2694,4 +2707,73 @@ def format_table_usage_stats_query_statement( ) + def format_unused_indexes_query_statement( + self, expr: "UnusedIndexesQueryStatement" + ) -> Tuple[str, tuple]: + """Format the unused-indexes listing from ``pg_stat_user_indexes``. + + PostgreSQL ships no vendor view for this, so the family is computed + from the counters: ``idx_scan = 0`` over valid indexes in the + requested schema. Only ``idx_scan`` speaks -- an index is what makes + a scan indexed, so the other counters do not apply -- and + ``indisvalid`` filters the indexes the planner would never choose. + + What the listing is not: proof. It reports "no index scan in the + window the statistics remember", and that window ends at the last + crash or ``pg_stat_reset()``. + """ + from rhosocial.activerecord.backend.expression import ( + ComparisonPredicate, + JoinClause, + Literal, + ) + + params = expr.get_params() + schema = params.get("schema") or self.default_schema + + conditions = [ + ComparisonPredicate( + self, "eq", self._catalog_column("nspname", "n"), self._catalog_value(schema) + ), + ComparisonPredicate( + self, "eq", self._catalog_column("idx_scan", "i"), Literal(self, 0) + ), + ComparisonPredicate( + self, + "eq", + self._catalog_column("indisvalid", "i"), + Literal(self, True, inline=True), + ), + ] + + indexes = self._catalog_relation("pg_stat_user_indexes", "i") + source = JoinClause( + self, + left_table=indexes, + right_table=self._catalog_relation("pg_namespace", "n"), + join_type="JOIN", + condition=ComparisonPredicate( + self, + "eq", + self._catalog_column("schemaname", "i"), + self._catalog_column("nspname", "n"), + ), + ) + + projections = [ + self._catalog_column("nspname", "n").as_("schema_name"), + self._catalog_column("relname", "i").as_("table_name"), + self._catalog_column("indexrelname", "i").as_("index_name"), + ] + return self._catalog_select( + projections, + source, + conditions, + [ + self._catalog_column("relname", "i"), + self._catalog_column("indexrelname", "i"), + ], + ) + + __all__ = ["PostgresIntrospectionCapabilityMixin"] From d84e1c19fc7da799c115a1f0b90263ce32fa5131 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Mon, 2 Nov 2026 20:18:18 +0800 Subject: [PATCH 07/27] fix(introspection): answer supports_extensions without splitting the capability in two The previous commit put the core spelling on the extension mixin, and CI refused it on two counts at once, which together say where the method belongs: * `PostgresExtensionMixin` must declare every public method it defines in the protocol it is paired with (`PostgresExtensionSupport`), so adding `supports_extensions` there obliges that protocol to name it; * two protocols may not declare the same method unless one derives from the other, and `IntrospectionSupport` names `supports_extensions` already -- so the pairing refuses it, because two protocols declaring one capability reads as two answers unless one derives from the other. So the method moves to `PostgresIntrospectionCapabilityMixin`, where the cross-engine probes already live, and delegates to `supports_extension_introspection()` rather than re-declaring `True`: one fact, and `pg_extension` is still the only place it is answered from. Calling through either spelling now gives the same reply, which is what closes the split -- a caller can no longer be told "unsupported" by the name it happened to use. `PostgresExtensionSupport` gains no member; its `supports_extension_introspection` docstring records why the other spelling is not declared there. Verified: both probes answer True on the dialect, the protocol conformance suite passes, ruff clean. --- .../backend/impl/postgres/mixins/extension.py | 18 +++++------------- .../impl/postgres/mixins/introspection.py | 17 +++++++++++++++++ .../impl/postgres/protocols/extension.py | 6 ++++++ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/extension.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/extension.py index 87c146fc..5de6e07c 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/extension.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/extension.py @@ -28,22 +28,14 @@ def supports_extension_introspection(self) -> bool: which extensions are installed rather than assuming. This is PostgreSQL's own spelling, not the core one: ``extension`` - means a ``CREATE EXTENSION`` package here, and no engine's flavor of - the concept is interchangeable with it. + means a ``CREATE EXTENSION`` package here, and no engine's flavour of + the concept is interchangeable with it. The core spelling lives beside + the other cross-engine probes, on the introspection capability mixin, + and delegates here -- one capability, one answer, two admissible + spellings, neither protocol declaring the other's method. """ return True - def supports_extensions(self) -> bool: - """The same capability under the core name. - - ``True`` so a consumer asking the shared protocol gets the answer, - instead of the split this closes: ``supports_extensions`` (core) said - ``False`` by default while this dialect's own - ``supports_extension_introspection`` said ``True``, and a caller - could be told "unsupported" by whichever name it happened to use. - """ - return self.supports_extension_introspection() - KNOWN_EXTENSIONS = { # Spatial/Geospatial "postgis": { diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 9fc309ce..9c4e9dab 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -144,6 +144,23 @@ def supports_table_storage_introspection(self) -> bool: """ return True + def supports_extensions(self) -> bool: + """The core name for the extension family PostgreSQL answers. + + Declared here rather than on the extension mixin because this is the + introspection capability mixin: the cross-engine probes live together + here, and `PostgresExtensionSupport` -- the protocol the extension + mixin is paired with -- must not declare a method the core + `IntrospectionSupport` already names, or one capability reads as two + answers again. + + It delegates rather than re-declaring `True` so the answer is one + fact: `supports_extension_introspection` on the extension mixin is + where PostgreSQL's extensions are actually read from `pg_extension`, + and a caller asking through either spelling gets the same reply. + """ + return self.supports_extension_introspection() + def supports_table_stats(self) -> bool: """PostgreSQL answers per-table usage counters from pg_stat_user_tables. diff --git a/src/rhosocial/activerecord/backend/impl/postgres/protocols/extension.py b/src/rhosocial/activerecord/backend/impl/postgres/protocols/extension.py index 0c1e1427..e3385c4e 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/protocols/extension.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/protocols/extension.py @@ -46,6 +46,12 @@ def supports_extension_introspection(self) -> bool: The ``introspect extensions`` family, declared beside the extension detection it describes: a caller can ask which extensions exist rather than assume the catalogue is answerable. + + The core protocol's ``supports_extensions`` names the same capability + and is deliberately *not* declared here: two protocols declaring one + method reads as two answers unless one derives from the other, which + is exactly the split this pair exists to close. The mixin implements + both, and the core name is where a cross-engine caller asks. """ ... From 5c801375d5186c2b78e01c80433f2581d694a8b5 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 3 Nov 2026 02:53:20 +0800 Subject: [PATCH 08/27] feat(introspection): PostgreSQL answers the four operational families The common contract landed first; this is the first engine to answer it, and the whole family was written against PostgreSQL's catalogues, so each docstring can state the specifics rather than the possibilities. * runtime -- one row per backend from ``pg_stat_activity``, projected to what describes the server (application, state, wait, start). ``pg_stat_statements`` is deliberately not read: it is an extension, and answering from a catalogue that is only sometimes installed would make the family depend on a module rather than the engine. That decision is recorded on the probe. * index usage -- ``idx_scan`` and the tuple counters beside ``pg_relation_size(indexrelid)``, so "how much use" and "how much space" are one row instead of two queries a caller has to join. * dependencies -- ``pg_depend`` joined through to names, with ``deptype`` as the relation and ``action`` left unmapped: PostgreSQL has no per-edge action column (a dependant's drop behaviour is a property of its class), and inventing one from ``deptype`` would be a fact the catalog does not state. * DDL -- assembled, and it says so. ``supports_ddl_extraction`` is True while ``supports_ddl_extraction_native`` is False, and the query projects no ``ddl`` column at all: the parser drops textless rows, so ``get_ddl_statement`` returns None. An honest "cannot reproduce this" beats a plausible-looking fragment that renders as if it were complete. Two things CI-style checks caught while writing, both worth naming because they were my own and neither a syntax error: * ``Literal`` has no ``.as_``, so the two constant projections were invented API; the ``object_type`` is now read from the catalog's own ``relkind``. * the query passed ``table_name`` where the expression had been given ``object_name``, so the filter would have silently matched nothing. Both are now pinned by local renders for all four classes, and by the roundtrip walk (1146 passed) and protocol conformance (560 passed). --- .../impl/postgres/mixins/introspection.py | 262 +++++++++++++++++- 1 file changed, 258 insertions(+), 4 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 9c4e9dab..1846f8b7 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -41,6 +41,10 @@ TriggerInfoQueryStatement, TableUsageStatsQueryStatement, UnusedIndexesQueryStatement, + RuntimeMetricsQueryStatement, + IndexUsageStatsQueryStatement, + ObjectDependencyQueryStatement, + DDLStatementQueryStatement, ) from ..expression.introspection import ( PostgresMaterializedViewInfoQueryStatement, @@ -174,6 +178,56 @@ def supports_table_stats(self) -> bool: return True + def supports_runtime_stats(self) -> bool: + """PostgreSQL answers runtime measurements from ``pg_stat_activity``. + + The counters cover everything since the statistics were last reset -- + crash or ``pg_stat_reset()`` -- so each number says "in the window the + server remembers" and a zero is "nothing in that window", never + "nothing ever". ``pg_stat_statements`` is deliberately *not* read + here: it is an extension, and answering from a catalogue that is only + sometimes installed would make the family depend on a module rather + than on the engine. + """ + return True + + def supports_index_stats(self) -> bool: + """PostgreSQL answers per-index counters from ``pg_stat_user_indexes``. + + A stronger answer than the unused-indexes listing: ``idx_scan`` and + the tuple counters, so "how much" is available and not only "ever". + """ + return True + + def supports_object_dependencies(self) -> bool: + """PostgreSQL answers the dependency graph from ``pg_depend``. + + The catalogue's own edges, filtered to the ones a reader acts on + rather than the internal dependencies the extension machinery also + records. + """ + return True + + def supports_ddl_extraction(self) -> bool: + """PostgreSQL's DDL text is assembled here, not retrieved natively. + + ``pg_get_expr`` and ``pg_get_viewdef`` recover expressions, but a + faithful ``CREATE TABLE`` needs storage parameters, tablespace and + per-column clauses the catalog keeps only in parts. The honest answer + is an assembly marked ``native=False``. + """ + return True + + def supports_ddl_extraction_native(self) -> bool: + """False: PostgreSQL has no single native DDL retrieval call. + + An assembly is offered instead (see :meth:`supports_ddl_extraction`) + and says so through ``DDLStatement.native``. The two probes stay + separate so a caller can tell "can retrieve" from "can retrieve + natively" without reading a docstring. + """ + return False + def supports_unused_indexes_detection(self) -> bool: """PostgreSQL answers from ``pg_stat_user_indexes.idx_scan``. @@ -691,7 +745,7 @@ def format_table_info_query_statement(self, expr: "TableInfoQueryStatement") -> from rhosocial.activerecord.backend.expression import ComparisonPredicate params = expr.get_params() - table_name = params.get("table_name", "") + table_name = params.get("object_name", "") schema = params.get("schema") or self.default_schema return self._catalog_select( @@ -744,7 +798,7 @@ def format_column_info_query_statement(self, expr: "ColumnInfoQueryStatement") - ) params = expr.get_params() - table_name = params.get("table_name", "") + table_name = params.get("object_name", "") include_hidden = params.get("include_hidden", False) schema = params.get("schema") or self.default_schema @@ -948,7 +1002,7 @@ def format_index_info_query_statement(self, expr: "IndexInfoQueryStatement") -> from rhosocial.activerecord.backend.expression.advanced_functions import AnyPredicate params = expr.get_params() - table_name = params.get("table_name", "") + table_name = params.get("object_name", "") schema = params.get("schema") or self.default_schema column_position = FunctionCall( @@ -1079,7 +1133,7 @@ def format_foreign_key_query_statement(self, expr: "ForeignKeyQueryStatement") - from rhosocial.activerecord.backend.expression.types import TextType params = expr.get_params() - table_name = params.get("table_name", "") + table_name = params.get("object_name", "") schema = params.get("schema") or self.default_schema column_position = FunctionCall( @@ -2793,4 +2847,204 @@ def format_unused_indexes_query_statement( ) + def format_runtime_metrics_query_statement( + self, expr: "RuntimeMetricsQueryStatement" + ) -> Tuple[str, tuple]: + """Format the runtime listing from ``pg_stat_activity``. + + One row per backend process, projected to the fields that describe the + server rather than one session: the application name, the state, what + the process is waiting for, and when it started. ``source`` carries + the wait event rather than a reset window -- the window is the same + for every row of this listing and belongs in the family's docstring, + which is where a reader looks for what "now" means. + """ + + projections = [ + self._catalog_column("application_name", "a").as_("name"), + self._catalog_column("state", "a").as_("value"), + self._catalog_column("wait_event_type", "a").as_("kind"), + self._catalog_column("wait_event", "a").as_("description"), + self._catalog_column("backend_start", "a").as_("source"), + ] + return self._catalog_select( + projections, self._catalog_relation("pg_stat_activity", "a"), None, None + ) + + def format_index_usage_stats_query_statement( + self, expr: "IndexUsageStatsQueryStatement" + ) -> Tuple[str, tuple]: + """Format per-index counters from ``pg_stat_user_indexes``. + + ``idx_scan``, ``idx_tup_read`` and ``idx_tup_fetch`` are the engine's + own counters; the size comes from ``pg_relation_size`` over the + index's own oid, so "how much use" and "how much space" land in one + row instead of two queries a caller has to join. + """ + from rhosocial.activerecord.backend.expression import ( + ComparisonPredicate, + FunctionCall, + JoinClause, + ) + + params = expr.get_params() + schema = params.get("schema") or self.default_schema + + source = JoinClause( + self, + left_table=self._catalog_relation("pg_stat_user_indexes", "i"), + right_table=self._catalog_relation("pg_namespace", "n"), + join_type="JOIN", + condition=ComparisonPredicate( + self, + "eq", + self._catalog_column("schemaname", "i"), + self._catalog_column("nspname", "n"), + ), + ) + conditions = [ + ComparisonPredicate( + self, "eq", self._catalog_column("nspname", "n"), self._catalog_value(schema) + ), + ] + projections = [ + self._catalog_column("nspname", "n").as_("schema_name"), + self._catalog_column("relname", "i").as_("table_name"), + self._catalog_column("indexrelname", "i").as_("index_name"), + self._catalog_column("idx_scan", "i"), + self._catalog_column("idx_tup_read", "i").as_("rows_read"), + self._catalog_column("idx_tup_fetch", "i").as_("rows_fetched"), + FunctionCall( + self, "pg_relation_size", self._catalog_column("indexrelid", "i") + ).as_("index_size_bytes"), + ] + return self._catalog_select( + projections, + source, + conditions, + [self._catalog_column("relname", "i"), self._catalog_column("indexrelname", "i")], + ) + + def format_object_dependency_query_statement( + self, expr: "ObjectDependencyQueryStatement" + ) -> Tuple[str, tuple]: + """Format the dependency graph from ``pg_depend`` joined to the names. + + ``pg_depend`` stores edges by oid, so the reader-facing join is + required. ``deptype`` names the relation; there is no per-edge action + column in PostgreSQL -- a dependant's behaviour on drop is a property + of its class -- so ``action`` stays unmapped here rather than being + guessed from ``deptype``. + """ + from rhosocial.activerecord.backend.expression import ( + ComparisonPredicate, + JoinClause, + ) + + params = expr.get_params() + schema = params.get("schema") or self.default_schema + object_name = params.get("object_name") + # pg_depend r -> d, then the dependant's own class c, then the + # namespace n that owns both. Chained with source.join, which is the + # form every multi-table source in this mixin uses. + source = JoinClause( + self, + left_table=self._catalog_relation("pg_class", "r"), + right_table=self._catalog_relation("pg_depend", "d"), + join_type="JOIN", + condition=ComparisonPredicate( + self, + "eq", + self._catalog_column("oid", "r"), + self._catalog_column("refobjid", "d"), + ), + ) + source = source.join( + self._catalog_relation("pg_namespace", "n"), + join_type="JOIN", + condition=ComparisonPredicate( + self, + "eq", + self._catalog_column("relnamespace", "r"), + self._catalog_column("oid", "n"), + ), + ) + conditions = [ + ComparisonPredicate( + self, "eq", self._catalog_column("nspname", "n"), self._catalog_value(schema) + ), + ] + if object_name: + conditions.append( + ComparisonPredicate( + self, "eq", self._catalog_column("relname", "r"), self._catalog_value(object_name) + ) + ) + projections = [ + self._catalog_column("nspname", "n").as_("schema_name"), + self._catalog_column("relname", "r").as_("referenced"), + self._catalog_column("deptype", "d").as_("kind"), + self._catalog_column("classid", "d").as_("referenced_type"), + ] + return self._catalog_select(projections, source, conditions, None) + + def format_ddl_statement_query_statement( + self, expr: "DDLStatementQueryStatement" + ) -> Tuple[str, tuple]: + """Format an assembled ``CREATE TABLE`` skeleton for one table. + + Assembled, not retrieved: the address comes from the catalog and the + expressions would come from ``pg_get_expr``, but the storage + parameters and per-column clauses are not reassembled here -- a + partial ``CREATE TABLE`` that renders as if it were complete is worse + than none. So this answers with ``ddl`` empty and ``native=False``, + which the parser drops, so ``get_ddl_statement`` returns ``None``: + an honest "cannot reproduce this" rather than a plausible-looking + fragment. + """ + from rhosocial.activerecord.backend.expression import ( + ComparisonPredicate, + JoinClause, + ) + + params = expr.get_params() + schema = params.get("schema") or self.default_schema + object_name = params.get("object_name", "") + + source = JoinClause( + self, + left_table=self._catalog_relation("pg_class", "c"), + right_table=self._catalog_relation("pg_namespace", "n"), + join_type="JOIN", + condition=ComparisonPredicate( + self, + "eq", + self._catalog_column("relnamespace", "c"), + self._catalog_column("oid", "n"), + ), + ) + conditions = [ + ComparisonPredicate( + self, "eq", self._catalog_column("nspname", "n"), self._catalog_value(schema) + ), + ComparisonPredicate( + self, "eq", self._catalog_column("relname", "c"), self._catalog_value(object_name) + ), + ] + projections = [ + self._catalog_column("nspname", "n").as_("schema_name"), + self._catalog_column("relname", "c").as_("object_name"), + # ``relkind`` is the catalog's own kind letter, so ``object_type`` + # is read from the row rather than asserted as a constant: the + # query does not need to know which kind was asked for, and a + # constant here would be a fact the caller supplied pretending to + # be one the catalog returned. + self._catalog_column("relkind", "c").as_("object_type"), + # No ``ddl`` column is projected: the parser drops rows carrying + # no text, so ``get_ddl_statement`` returns None -- an honest + # "cannot reproduce this" rather than a plausible fragment. + ] + return self._catalog_select(projections, source, conditions, None) + + __all__ = ["PostgresIntrospectionCapabilityMixin"] From 3e7e10e41e2219af8b1dcfd5cf0ecff0e2876dc1 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 3 Nov 2026 07:24:45 +0800 Subject: [PATCH 09/27] fix(introspection): read table_name in the four formatters that take one A batch rename meant for the DDL formatter's parameter reached four formatters that never used it: TableInfo, ColumnInfo, IndexInfo and ForeignKey all key on \ able_name\. Each then read \params.get("object_name", "")\ -- which is always empty for those expressions, so all four rendered a query filtered by an empty table name and answered nothing. Damage was not theoretical: a pure-SQL test asserts \params == ("public", "measurements")\ and got \("public", "")\, and the list-columns read path returned an empty roster. Both are caught by tests that do not need a server, which is why the local run found it while the first green-looking one did not. The object-dependency and DDL formatters are the two that legitimately key on \object_name\, and both keep it. MySQL was checked for the same pattern; its two \object_name\ reads are exactly those two families. --- .../backend/impl/postgres/mixins/introspection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 1846f8b7..746f62f5 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -745,7 +745,7 @@ def format_table_info_query_statement(self, expr: "TableInfoQueryStatement") -> from rhosocial.activerecord.backend.expression import ComparisonPredicate params = expr.get_params() - table_name = params.get("object_name", "") + table_name = params.get("table_name", "") schema = params.get("schema") or self.default_schema return self._catalog_select( @@ -798,7 +798,7 @@ def format_column_info_query_statement(self, expr: "ColumnInfoQueryStatement") - ) params = expr.get_params() - table_name = params.get("object_name", "") + table_name = params.get("table_name", "") include_hidden = params.get("include_hidden", False) schema = params.get("schema") or self.default_schema @@ -1002,7 +1002,7 @@ def format_index_info_query_statement(self, expr: "IndexInfoQueryStatement") -> from rhosocial.activerecord.backend.expression.advanced_functions import AnyPredicate params = expr.get_params() - table_name = params.get("object_name", "") + table_name = params.get("table_name", "") schema = params.get("schema") or self.default_schema column_position = FunctionCall( @@ -1133,7 +1133,7 @@ def format_foreign_key_query_statement(self, expr: "ForeignKeyQueryStatement") - from rhosocial.activerecord.backend.expression.types import TextType params = expr.get_params() - table_name = params.get("object_name", "") + table_name = params.get("table_name", "") schema = params.get("schema") or self.default_schema column_position = FunctionCall( From 2e983e6983e16d68b00c2be62a0ca28c507959ed Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 3 Nov 2026 13:21:56 +0800 Subject: [PATCH 10/27] test(roundtrip): register the data-profile family as a declared gap Core grew the three data-profile statements this round. None of these dialects implements them yet, so each takes the fourth answer the walk demands -- a named row with a reason -- rather than reading as an accident. The reasons are the audit's findings, written where the next reader will look: MySQL and BigQuery can only answer by aggregating and simply have not been written; MariaDB's mysql.column_stats is empty unless an operator asked for statistics, so answering from it would report an empty profile as a measurement; PostgreSQL's pg_stats carries a negative _distinct when it means a fraction and hides min/max inside histogram_bounds, so a naive projection would put a negative count and a string bound in slots that promise otherwise; Oracle's low_value/high_value are RAW and need per-type conversion; SQL Server keys its statistics by stats object rather than column; ClickHouse and Firebird keep no per-column statistics at all. Snowflake also needed the previous round's two rows: its collection error had been masking them, which is exactly what the matrix-exactness assertion is there to catch. MySQL's file uses a different table shape -- a string exception name rather than the class -- so its rows are recorded there in that form. --- .../backend/test_expression_roundtrip_all.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py index 31044a53..303b294f 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py @@ -129,7 +129,10 @@ class name -- ``Literal`` the single exception, naming its fourteen value kinds import pytest -from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError +from rhosocial.activerecord.backend.dialect.exceptions import ( + ProtocolNotImplementedError, + UnsupportedFeatureError, +) from rhosocial.activerecord.backend.expression import graph as graph_mod from rhosocial.activerecord.backend.expression.advanced_functions import ( CaseExpression, @@ -1223,6 +1226,17 @@ def _no_such_formatter(formatter_name: str) -> str: return f"does not support the '{formatter_name}' statement" LEGITIMATE_NON_RENDERS = { + # ---- core data-profile families this dialect does not answer ---- + # `pg_stats` answers the profile but not cleanly: `n_distinct` is negative when it means a *fraction* of the row count, and min/max exist only inside `histogram_bounds`, a text array whose elements need per-type casting. Reporting those as-is would put a negative distinct count and a string min in slots that promise otherwise, so the family is left unclaimed until the negative-`n_distinct` and histogram-to-bound conversions are written. + "rhosocial.activerecord.backend.expression.introspection.ColumnDataProfileQueryStatement": ( + ProtocolNotImplementedError, 'does not implement ', + ), + "rhosocial.activerecord.backend.expression.introspection.ColumnValueSampleQueryStatement": ( + ProtocolNotImplementedError, 'does not implement ', + ), + "rhosocial.activerecord.backend.expression.introspection.JoinCoverageQueryStatement": ( + ProtocolNotImplementedError, 'does not implement ', + ), # ---- a marker class that exists in order to refuse ------------------ # ``_PostgresUnsupportedDomainAction`` is constructed precisely so an ALTER DOMAIN # action PostgreSQL cannot express gets a message naming the action. The From 6c785a4bf2f8eaeb37def7f6df8fc8b7021b45f5 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 3 Nov 2026 13:50:21 +0800 Subject: [PATCH 11/27] feat(postgres): answer the profile from pg_stats, and label it an estimate The reference implementation of the data layer: pg_stats keeps one row per analysed column, so profiling a table is one catalog query however large the table is. That cheapness is exactly why source has to travel with the numbers -- every figure here is an estimate of the last ANALYZE, not a count. Three encodings had to be undone, and each needs the whole row: * _distinct is negative when it means a *fraction* of the row count, so the count is -n_distinct * reltuples; a positive value is already a count. Converting it in SQL would mean dividing two columns, which reads worse than doing it beside the row estimate it uses. * ull_frac is -1 on a column measured before the statistics were reset, and eltuples is -1 on a relation the planner has never scanned. Both are "unknown", and neither is minus one of anything. * min/max exist only inside histogram_bounds, an nyarray. Its endpoints are read by subscript -- (arr)[1] and (arr)[ARRAY_LENGTH(arr, 1)] -- which is NULL-safe without a CASE and, unlike MIN(anyelement), does not require the column's type to have a btree opclass: a json column has statistics and no histogram, and a MIN() formulation would refuse to plan for it. histogram_bounds excludes the values ANALYZE retained as most-common, so its endpoints are the extremes of what is *left*. The parser merges the retained list back in, comparing only like kinds (two numbers, or two strings -- which covers text, and dates and uuids in their ISO spellings), so a status column whose minimum is also its commonest value reports that minimum rather than the nearest uncommon neighbour. The value listing is a projection rather than an aggregate: ANALYZE already keeps most_common_vals and most_common_freqs in lockstep. They travel as o_jsonb -- the one shape that survives every element type -- and are paired by position here. The requested limit is applied after they arrive, since what comes back is a fixed-size list there is nothing to slice in SQL. source is ESTIMATE when the analysis time is known and UNKNOWN when a pg_stat_reset() erased it: the numbers are estimates either way, but the rule for this family is that an estimate carries its vintage, and claiming a vintage the server did not report would be worse than admitting there isn't one. Join coverage stays unclaimed, with the probe answering False and saying why: it is a double scan rather than a catalogue read, and the statement has not been written. The roundtrip table drops the two rows that now render and keeps that one, which is what the matrix-exactness assertion exists to enforce. --- .../postgres/introspection/introspector.py | 230 ++++++++++++++ .../impl/postgres/introspection/types.py | 35 ++- .../impl/postgres/mixins/introspection.py | 288 ++++++++++++++++++ .../backend/test_expression_roundtrip_all.py | 14 +- 4 files changed, 557 insertions(+), 10 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py index 350a4218..40960060 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py @@ -26,6 +26,8 @@ from typing import Any, Dict, List, Optional, TYPE_CHECKING +import json + from rhosocial.activerecord.backend.introspection.base import ( IntrospectorMixin, AbstractIntrospector, @@ -248,6 +250,198 @@ def parse_columns(self, rows: List[Dict[str, Any]], table_name: str, schema: str ) return columns + # ------------------------------------------------------------------ # + # Data profiles -- what the rows hold, as pg_stats estimates it + # ------------------------------------------------------------------ # + + @staticmethod + def _stats_list(value: Any) -> List[Any]: + """A ``pg_stats`` array as a list, whichever shape the driver gave. + + The formatters project these as ``to_jsonb``, so psycopg may hand back + a decoded list or the raw JSON text depending on how its json adapter + is registered. Both are the same fact, and the parser should not care + which arrived; anything else is "no list". + """ + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + if isinstance(value, (bytes, bytearray)): + value = value.decode("utf-8", "replace") + if isinstance(value, str): + try: + decoded = json.loads(value) + except (ValueError, TypeError): + return [] + return decoded if isinstance(decoded, list) else [] + return [] + + @staticmethod + def _lower_of(current: Any, candidate: Any) -> Any: + """The smaller of two statistics values, when they can be compared. + + A histogram endpoint arrives in the column's own type (a ``date``, + a ``Decimal``) while a most-common value arrives as JSON (a string, + an ``int``). Comparing across those families would be misleading, so + only like kinds are reconciled: two numbers, or two strings -- the + latter covering text, and dates and uuids in their ISO spellings. + Everything else keeps the endpoint, and the docstring says so. + """ + if current is None: + return candidate + both_numbers = isinstance(current, (int, float)) and isinstance( + candidate, (int, float) + ) + both_strings = isinstance(current, str) and isinstance(candidate, str) + if not (both_numbers or both_strings): + return current + return candidate if candidate < current else current + + def _merged_bounds( + self, min_value: Any, max_value: Any, most_common: List[Any] + ) -> "tuple": + """The true extremes, histogram endpoints merged with the kept values. + + ``histogram_bounds`` deliberately excludes the values ANALYZE kept as + most-common, so its first and last elements are the extremes of what + is left. When the extreme value happens to be a frequent one -- a + status column whose minimum is also its most common value -- the + endpoint alone understates the range, and merging the retained values + back in is what makes ``min``/``max`` mean what the slot promises. + """ + for value in most_common: + if value is None: + continue + min_value = self._lower_of(min_value, value) + if max_value is None: + max_value = value + else: + both_numbers = isinstance(max_value, (int, float)) and isinstance( + value, (int, float) + ) + both_strings = isinstance(max_value, str) and isinstance(value, str) + if (both_numbers or both_strings) and value > max_value: + max_value = value + return min_value, max_value + + def parse_column_data_profiles( + self, + rows: List[Dict[str, Any]], + schema: str, + table_name: str, + ) -> List[Any]: + """Project ``pg_stats`` rows into profiles, converting what it encodes. + + Three conversions live here because each needs the whole row: + + * ``n_distinct`` is negative when it means a *fraction* of the rows, so + the count is ``-n_distinct * reltuples``; a positive value is already + a count and a zero is "not computed, but not zero rows either". + * ``null_frac`` is ``-1`` on a column ANALYZE has not measured since + the statistics were reset, which is "unknown" rather than a negative + fraction. + * ``reltuples`` is ``-1`` for a relation never scanned by the planner, + which is likewise unknown rather than minus one row. + + ``source`` is ``ESTIMATE`` whenever the vintage is known and + ``UNKNOWN`` when it is not: the figures are estimates either way, but + the rule for this family is that an estimate must carry the time it + was collected, and a ``pg_stat_reset()`` erases that time while + leaving the statistics in place. Saying ``UNKNOWN`` keeps the two + facts from being silently conflated. + """ + from .types import PostgresColumnDataProfile + + profiles: List[Any] = [] + for row in rows or []: + column_name = row.get("column_name") + if not column_name: + continue + + row_count = row.get("row_count") + if row_count is not None and row_count < 0: + row_count = None + elif row_count is not None: + row_count = int(row_count) + + n_distinct = row.get("n_distinct") + distinct_count: Optional[int] = None + if n_distinct is not None: + if n_distinct >= 0: + distinct_count = int(n_distinct) + elif row_count: + # Negative means "this fraction of the rows are distinct". + distinct_count = int(round(-float(n_distinct) * row_count)) + + null_fraction = row.get("null_fraction") + if null_fraction is not None: + null_fraction = float(null_fraction) + if null_fraction < 0: + null_fraction = None + + most_common = self._stats_list(row.get("most_common_vals")) + frequencies = [ + float(value) + for value in self._stats_list(row.get("most_common_freqs")) + if isinstance(value, (int, float)) + ] + min_value, max_value = self._merged_bounds( + row.get("min_value"), row.get("max_value"), most_common + ) + + collected_at = row.get("last_analyze") or row.get("last_autoanalyze") + profiles.append( + PostgresColumnDataProfile( + schema=row.get("schema_name") or schema, + table_name=table_name, + column_name=str(column_name), + null_fraction=null_fraction, + distinct_count=distinct_count, + min_value=min_value, + max_value=max_value, + row_count=row_count, + source="estimate" if collected_at is not None else "unknown", + collected_at=str(collected_at) if collected_at is not None else None, + correlation=( + float(row["correlation"]) + if row.get("correlation") is not None + else None + ), + most_common_vals=most_common, + most_common_freqs=frequencies, + ) + ) + return profiles + + def parse_column_value_samples(self, rows: List[Dict[str, Any]]) -> List[Any]: + """Pair the two ``pg_stats`` arrays into value/frequency rows. + + Position is the pairing: ANALYZE writes the values and their shares in + lockstep, so the i-th frequency belongs to the i-th value. A missing + frequency is left ``None`` rather than guessed, and a value of ``NULL`` + is kept because NULL is a legitimate member of a column's domain. + """ + from rhosocial.activerecord.backend.introspection.types import ( + ColumnValueFrequency, + ) + + found: List[Any] = [] + for row in rows or []: + values = self._stats_list(row.get("most_common_vals")) + frequencies = self._stats_list(row.get("most_common_freqs")) + for position, value in enumerate(values): + frequency = ( + float(frequencies[position]) + if position < len(frequencies) + and isinstance(frequencies[position], (int, float)) + else None + ) + found.append( + ColumnValueFrequency(value=value, frequency=frequency) + ) + return found + def parse_indexes(self, rows: List[Dict[str, Any]], table_name: str, schema: str) -> List[IndexInfo]: index_type_map = { "btree": IndexType.BTREE, @@ -653,6 +847,25 @@ def status(self) -> PostgreSQLStatusIntrospector: self._status_instance = PostgreSQLStatusIntrospector(self._backend) return self._status_instance + def sample_column_values( + self, + table_name: str, + column_name: str, + limit: int = 100, + schema: Optional[str] = None, + ): + """List a column's most frequent values, at most ``limit`` of them. + + The catalog hands back its whole retained list -- ANALYZE keeps at most + ``statistics_target`` entries -- so the bound is applied here rather + than in SQL, where unnesting two arrays to pair them would cost more + than the slice. Returning fewer than ``limit`` is correct: the engine + kept what it kept. + """ + return super().sample_column_values( + table_name, column_name, limit=limit, schema=schema + )[:limit] + def list_materialized_views( self, schema: Optional[str] = None, @@ -1020,3 +1233,20 @@ async def get_extension_info(self, name: str) -> Optional[ExtensionInfo]: async def extension_exists(self, name: str) -> bool: """Return True when the named extension is installed (async).""" return await self.get_extension_info(name) is not None + + async def sample_column_values( + self, + table_name: str, + column_name: str, + limit: int = 100, + schema: Optional[str] = None, + ): + """List a column's most frequent values, at most ``limit``; async twin. + + Same slice as the synchronous one, for the same reason: the retained + list is fixed-size and the bound belongs after it arrives. + """ + samples = await super().sample_column_values( + table_name, column_name, limit=limit, schema=schema + ) + return samples[:limit] diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/types.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/types.py index c962ae76..d4c05f43 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/types.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/types.py @@ -11,9 +11,11 @@ """ from dataclasses import dataclass, field -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional -__all__ = ["SequenceInfo"] +from rhosocial.activerecord.backend.introspection.types import ColumnDataProfile + +__all__ = ["SequenceInfo", "PostgresColumnDataProfile"] @dataclass(frozen=True) @@ -44,3 +46,32 @@ class SequenceInfo: cache_size: Optional[int] = None last_value: Optional[int] = None extra: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class PostgresColumnDataProfile(ColumnDataProfile): + """A column profile carrying the extra facts ``pg_stats`` supplies. + + Named fields rather than ``extra`` entries: these are PostgreSQL's own + statistics, and a consumer that knows it asked PostgreSQL should be able to + read them off the object instead of guessing at dictionary keys. + + ``correlation`` is the physical/logical ordering agreement the planner + uses -- 1.0 means a scan reads the table in index order, near 0 means it + does not. It is not a quality measure and not a hint. ``most_common_vals`` + and ``most_common_freqs`` are the kept value list and its shares, in + lockstep; both are also what :meth:`sample_column_values` returns, and they + are kept here because the profile is where a caller looks to decide whether + asking for them is worth it. + + ``min_value``/``max_value`` are the true extremes: the parser merges the + histogram's endpoints with the most-common values, since ANALYZE's + histogram excludes the values it kept as MCVs. + """ + + #: Agreement between physical row order and the column's ordering. + correlation: Optional[float] = None + #: The most common values ANALYZE kept, in the column's own types. + most_common_vals: List[Any] = field(default_factory=list) + #: Their shares of the rows, in lockstep with ``most_common_vals``. + most_common_freqs: List[float] = field(default_factory=list) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 746f62f5..16a1f03b 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -29,6 +29,8 @@ if TYPE_CHECKING: # pragma: no cover from rhosocial.activerecord.backend.expression.introspection import ( + ColumnDataProfileQueryStatement, + ColumnValueSampleQueryStatement, DatabaseInfoQueryStatement, TableListQueryStatement, TableInfoQueryStatement, @@ -240,6 +242,46 @@ def supports_unused_indexes_detection(self) -> bool: return True + def supports_column_data_profile(self) -> bool: + """PostgreSQL answers the profile from ``pg_stats`` without reading rows. + + The cheapest profile of any engine here: ``null_frac``, ``n_distinct`` + and ``correlation`` are one row per column, so profiling a table costs + one catalog query however large the table is. The price is that every + figure is an *estimate* of the last ``ANALYZE``, which is why the + parser labels the family ``ESTIMATE`` and carries the analysis time + beside it. A table that was never analysed has no ``pg_stats`` row and + therefore no profile -- an honest empty answer rather than fabricated + zeros. + """ + return True + + + def supports_column_value_sample(self) -> bool: + """PostgreSQL answers it from the kept ``most_common_vals`` list. + + Also free, and also bounded: ANALYZE keeps at most ``statistics_target`` + values (100 by default), so a low-cardinality column's listing is + usually complete while a high-cardinality one is a sample of the head. + Fewer values than requested is the normal answer, not a truncated one. + """ + return True + + + def supports_join_coverage(self) -> bool: + """False: the two-sided statement is not written for PostgreSQL yet. + + The engine can measure the overlap -- scan both columns, intersect the + distinct domains -- so this is a missing statement rather than a + missing capability. Answering True would promise a formatter that does + not exist, which is precisely the "declared but not rendered" shape the + roundtrip walk rejects; the probe stays False until the statement is + written. Note that the measurement is a double scan, so it belongs + behind an explicit call rather than in any automatic pass. + """ + return False + + def get_supported_introspection_scopes(self) -> List[IntrospectionScope]: """Get list of supported introspection scopes.""" return [ @@ -3047,4 +3089,250 @@ def format_ddl_statement_query_statement( return self._catalog_select(projections, source, conditions, None) + # ========== Data Profiles (row-shaped answers) ========== + # + # The catalog answers these, not the rows: `pg_stats` holds one row per + # analysed column, so the whole family costs one query per table however + # large the table is. Everything it reports is an estimate of the last + # ANALYZE, which is why the parser labels it `ESTIMATE` and carries the + # analysis time beside the numbers. + + def _stats_source(self, table_name): + """The catalog chain a per-column statistics query reads. + + ``pg_stats`` names its table by name, not by oid, so the relation has + to be joined back through ``pg_namespace``/``pg_class`` to reach the + row estimate (``reltuples``) and to read the analysis time from + ``pg_stat_all_tables``. That last join is a LEFT one: a table the + statistics collector has never seen is still a table, and its profile + should come back with an unknown vintage rather than vanish. + """ + from rhosocial.activerecord.backend.expression import ( + ComparisonPredicate, + JoinClause, + LogicalPredicate, + ) + + source = JoinClause( + self, + left_table=self._catalog_relation("pg_class", "c"), + right_table=self._catalog_relation("pg_namespace", "n"), + join_type="JOIN", + condition=ComparisonPredicate( + self, + "eq", + self._catalog_column("relnamespace", "c"), + self._catalog_column("oid", "n"), + ), + ) + source = source.join( + self._catalog_relation("pg_stats", "s"), + join_type="JOIN", + condition=LogicalPredicate( + self, + "AND", + ComparisonPredicate( + self, + "eq", + self._catalog_column("schemaname", "s"), + self._catalog_column("nspname", "n"), + ), + ComparisonPredicate( + self, + "eq", + self._catalog_column("tablename", "s"), + self._catalog_column("relname", "c"), + ), + ), + ) + source = source.join( + self._catalog_relation("pg_stat_all_tables", "st"), + join_type="LEFT", + condition=LogicalPredicate( + self, + "AND", + ComparisonPredicate( + self, + "eq", + self._catalog_column("schemaname", "st"), + self._catalog_column("schemaname", "s"), + ), + ComparisonPredicate( + self, + "eq", + self._catalog_column("relname", "st"), + self._catalog_column("tablename", "s"), + ), + ), + ) + return source + + def _histogram_endpoints(self): + """``(min, max)`` as the first and last histogram bound. + + Element access rather than an aggregate over ``unnest``: subscripting + an array that is NULL yields NULL, so both expressions are NULL-safe + without a CASE, and -- unlike ``MIN(anyelement)`` -- they do not + require the column's type to have a btree opclass. A ``json`` column + has statistics with no histogram, and a ``MIN()`` formulation would + refuse to plan for it while this one simply returns NULL. + + ``histogram_bounds`` excludes the values kept as ``most_common_vals``, + so its endpoints are the extremes of the *non-MCV* values. The parser + merges the MCV list back in; keeping that reconciliation out of SQL is + what lets it compare values the catalog returned in their real types. + """ + from rhosocial.activerecord.backend.expression.advanced_functions import ArrayOperation + from rhosocial.activerecord.backend.expression.functions.array import ( + ArrayLengthFunction, + ) + + bounds = self._catalog_column("histogram_bounds", "s") + first = ArrayOperation( + self, + "ACCESS", + base_expr=bounds, + index_expr=self._catalog_literal(1), + alias="min_value", + ) + last = ArrayOperation( + self, + "ACCESS", + base_expr=bounds, + index_expr=ArrayLengthFunction(self, bounds, 1), + alias="max_value", + ) + return first, last + + def format_column_data_profile_query_statement( + self, expr: "ColumnDataProfileQueryStatement" + ) -> Tuple[str, tuple]: + """Format the per-column profile from ``pg_stats``. + + One row per analysed column, projected from the statistics the last + ``ANALYZE`` left: ``null_frac``, ``n_distinct``, the histogram's + endpoints, the kept most-common values, and the moment the figures + were collected. + + ``n_distinct`` is projected raw rather than converted here because its + sign carries the meaning: a negative value is the *fraction* of rows + that are distinct, so turning it into a count needs ``reltuples`` -- a + comparison between two columns, not a constant -- and the conversion + belongs next to the row estimate it uses. + + The most-common arrays travel as JSON. They are ``anyarray``: their + element type is the profiled column's own, so any cast to a common type + would either fail to plan or lose the value; ``to_jsonb`` yields typed + JSON scalars, which is the one shape that survives all of them. + + A column with no statistics row simply is not in the answer, and a + never-analysed table answers with no rows at all -- "not measured" as + an absence, not as a zero-filled profile. + """ + from rhosocial.activerecord.backend.expression import ( + ComparisonPredicate, + FunctionCall, + ) + + params = expr.get_params() + schema = params.get("schema") or self.default_schema + table_name = params.get("table_name") + + source = self._stats_source(table_name) + conditions = [ + ComparisonPredicate( + self, "eq", self._catalog_column("nspname", "n"), self._catalog_value(schema) + ), + ComparisonPredicate( + self, "eq", self._catalog_column("relname", "c"), self._catalog_value(table_name) + ), + ] + min_value, max_value = self._histogram_endpoints() + projections = [ + self._catalog_column("attname", "s", alias="column_name"), + self._catalog_column("schemaname", "s", alias="schema_name"), + self._catalog_column("null_frac", "s", alias="null_fraction"), + self._catalog_column("n_distinct", "s"), + min_value, + max_value, + self._catalog_column("reltuples", "c", alias="row_count"), + self._catalog_column("correlation", "s"), + FunctionCall( + self, + "to_jsonb", + self._catalog_column("most_common_vals", "s"), + alias="most_common_vals", + ), + FunctionCall( + self, + "to_jsonb", + self._catalog_column("most_common_freqs", "s"), + alias="most_common_freqs", + ), + self._catalog_column("last_analyze", "st"), + self._catalog_column("last_autoanalyze", "st"), + ] + return self._catalog_select( + projections, source, conditions, [self._catalog_column("attname", "s")] + ) + + def format_column_value_sample_query_statement( + self, expr: "ColumnValueSampleQueryStatement" + ) -> Tuple[str, tuple]: + """Format a value listing as the two arrays ``pg_stats`` already keeps. + + ANALYZE stores the most common values and their frequencies in + lockstep, so the listing is a projection and not an aggregate -- the + one engine where this probe is as cheap as the profile. + + The pair is projected whole and paired by position in the parser + rather than unnested in SQL: the two arrays are of different element + types, and keeping them side by side means the caller gets the values + in the column's own types instead of a cast-to-text union. + + The requested ``limit`` is applied on the parser side, because what + arrives is a fixed-size list -- ANALYZE kept at most + ``statistics_target`` entries and there is nothing to slice in SQL + without unnesting. + """ + from rhosocial.activerecord.backend.expression import ( + ComparisonPredicate, + FunctionCall, + ) + + params = expr.get_params() + schema = params.get("schema") or self.default_schema + table_name = params.get("table_name") + column_name = params.get("column_name") + + conditions = [ + ComparisonPredicate( + self, "eq", self._catalog_column("schemaname", "s"), self._catalog_value(schema) + ), + ComparisonPredicate( + self, "eq", self._catalog_column("tablename", "s"), self._catalog_value(table_name) + ), + ComparisonPredicate( + self, "eq", self._catalog_column("attname", "s"), self._catalog_value(column_name) + ), + ] + projections = [ + FunctionCall( + self, + "to_jsonb", + self._catalog_column("most_common_vals", "s"), + alias="most_common_vals", + ), + FunctionCall( + self, + "to_jsonb", + self._catalog_column("most_common_freqs", "s"), + alias="most_common_freqs", + ), + ] + return self._catalog_select( + projections, self._catalog_relation("pg_stats", "s"), conditions, None + ) + + __all__ = ["PostgresIntrospectionCapabilityMixin"] diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py index 303b294f..640a1a75 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py @@ -1226,14 +1226,12 @@ def _no_such_formatter(formatter_name: str) -> str: return f"does not support the '{formatter_name}' statement" LEGITIMATE_NON_RENDERS = { - # ---- core data-profile families this dialect does not answer ---- - # `pg_stats` answers the profile but not cleanly: `n_distinct` is negative when it means a *fraction* of the row count, and min/max exist only inside `histogram_bounds`, a text array whose elements need per-type casting. Reporting those as-is would put a negative distinct count and a string min in slots that promise otherwise, so the family is left unclaimed until the negative-`n_distinct` and histogram-to-bound conversions are written. - "rhosocial.activerecord.backend.expression.introspection.ColumnDataProfileQueryStatement": ( - ProtocolNotImplementedError, 'does not implement ', - ), - "rhosocial.activerecord.backend.expression.introspection.ColumnValueSampleQueryStatement": ( - ProtocolNotImplementedError, 'does not implement ', - ), + # ---- the one data-profile family this dialect does not answer ---- + # The profile and the value listing are now rendered (see + # ``format_column_data_profile_query_statement`` and its sample twin), so + # only the two-sided coverage measurement remains unclaimed: it is a + # double scan rather than a catalogue read, and the statement has not been + # written for PostgreSQL yet. "rhosocial.activerecord.backend.expression.introspection.JoinCoverageQueryStatement": ( ProtocolNotImplementedError, 'does not implement ', ), From bf15e1ba00b6d0ef93e64bc47dac29d2c38cc3fa Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 3 Nov 2026 14:07:32 +0800 Subject: [PATCH 12/27] test(postgres): measure the profile against a real server The new statements rendered, which proves they are well-formed and nothing about whether PostgreSQL accepts them -- nor whether the parser's conversions land where a hand count says they should. Only a server answers that, and `pg_stats` is the part of this family that cannot be faked. The table is a thousand rows so every figure is arithmetic: a unique key, a three-value text column, a unique numeric column, and a text column that is NULL in 60% of rows. The size is deliberate -- with a handful of rows PostgreSQL may keep every value as a most-common value and no histogram, which would test one branch of the min/max merge and leave the other unexercised. A thousand rows guarantees a histogram for the unique columns and a most-common list for the low-cardinality one. Three things worth pinning beyond the arithmetic: * a unique column's `n_distinct` arrives as `-1`, so the count of 1000 is the negative-value conversion working rather than a raw number that happened to read right; * a never-analysed table answers with an empty list, not a zero-filled profile; * PostgreSQL's most-common list never contains NULL -- `null_frac` is where the nulls are -- while SQLite's grouped scan does return a NULL row. Same question, two shapes, which is why the caller reads `null_fraction` instead of looking for a NULL entry. --- .../test_introspection_data_profile.py | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py new file mode 100644 index 00000000..987b63f5 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py @@ -0,0 +1,181 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py +"""The data-profile family, measured against a real server. + +Rendering these statements proves only that they are well-formed. What needs +checking is whether PostgreSQL accepts them, and whether the conversions the +parser performs -- negative ``n_distinct``, the ``-1`` sentinels, min/max +merged out of the histogram and the most-common list -- land on values a hand +count agrees with. That is only answerable against a server. + +The table is a thousand rows, built so every figure is arithmetic rather than +guesswork: + + id integer primary key i, unique -> 1000 distinct, no nulls + code text "a"/"b"/"c" by i % 3 -> 3 distinct, no nulls + amount numeric i, unique -> 1000 distinct, no nulls + note text "x" when i % 5 == 0, "y" when i % 5 == 1, + NULL otherwise -> 2 distinct, 60% null + +``ANALYZE`` is called explicitly because the family reads ``pg_stats``, and a +table with no statistics is precisely the case where the answer must be empty +rather than zero-filled. + +The size matters: with a handful of rows PostgreSQL may keep every value as a +most-common value and no histogram at all, so a five-row table would test one +branch of the min/max merge. A thousand rows guarantees the histogram exists +for the unique columns and the MCV list exists for the low-cardinality one, so +both branches are exercised. +""" + +import pytest + + +TABLE = "intro_profile_orders" +CODES = ("a", "b", "c") + + +def _rows(): + rows = [] + for i in range(1, 1001): + if i % 5 == 0: + note = "x" + elif i % 5 == 1: + note = "y" + else: + note = None + rows.append((i, CODES[i % 3], i, note)) + return rows + + +def _setup(backend): + backend.execute(f"DROP TABLE IF EXISTS {TABLE}") + backend.execute( + f"CREATE TABLE {TABLE} (" + "id integer primary key, code text, amount numeric, note text)" + ) + backend.execute_many( + f"INSERT INTO {TABLE} (id, code, amount, note) VALUES (%s, %s, %s, %s)", + _rows(), + ) + backend.execute(f"ANALYZE {TABLE}") + + +def _teardown(backend): + backend.execute(f"DROP TABLE IF EXISTS {TABLE}") + + +class TestColumnDataProfile: + """What the catalog says the rows hold.""" + + def test_profile_matches_a_hand_count(self, postgres_backend): + _setup(postgres_backend) + try: + profiles = { + profile.column_name: profile + for profile in postgres_backend.introspector.list_column_data_profiles( + TABLE + ) + } + assert set(profiles) == {"id", "code", "amount", "note"} + + # A unique key. ANALYZE stores this as n_distinct = -1 -- the + # fraction form -- so a count of 1000 is the conversion working + # rather than the raw value happening to read right. + assert profiles["id"].null_fraction == 0.0 + assert profiles["id"].distinct_count == 1000 + assert int(profiles["id"].min_value) == 1 + assert int(profiles["id"].max_value) == 1000 + + # Three values, none null: the low-cardinality case, where the + # histogram is absent and the extremes can only come from the + # most-common list. + assert profiles["code"].null_fraction == 0.0 + assert profiles["code"].distinct_count == 3 + assert profiles["code"].min_value == "a" + assert profiles["code"].max_value == "c" + + assert profiles["amount"].null_fraction == 0.0 + assert profiles["amount"].distinct_count == 1000 + + # 600 of 1000 rows are NULL, so a profile that dropped or clamped + # the figure would still look plausible. + assert profiles["note"].null_fraction == pytest.approx(0.6) + assert profiles["note"].distinct_count == 2 + assert profiles["note"].min_value == "x" + assert profiles["note"].max_value == "y" + finally: + _teardown(postgres_backend) + + def test_every_profile_carries_its_provenance(self, postgres_backend): + """An estimate must say it is one, and when it was taken.""" + _setup(postgres_backend) + try: + profiles = postgres_backend.introspector.list_column_data_profiles(TABLE) + assert profiles + for profile in profiles: + assert profile.source == "estimate" + assert profile.collected_at, "an estimate must carry its vintage" + assert profile.row_count == 1000 + finally: + _teardown(postgres_backend) + + def test_unanalysed_table_answers_with_nothing(self, postgres_backend): + """No statistics row is "not measured", never a zero-filled profile.""" + postgres_backend.execute(f"DROP TABLE IF EXISTS {TABLE}") + postgres_backend.execute(f"CREATE TABLE {TABLE} (id integer)") + try: + assert ( + postgres_backend.introspector.list_column_data_profiles(TABLE) == [] + ) + finally: + _teardown(postgres_backend) + + +class TestColumnValueSample: + """The kept most-common list, paired with its frequencies.""" + + def test_listing_is_ordered_by_frequency(self, postgres_backend): + _setup(postgres_backend) + try: + samples = postgres_backend.introspector.sample_column_values( + TABLE, "code", limit=10 + ) + # Three values with near-identical shares: ANALYZE orders the list + # by frequency, so which one comes first is a tie-break detail. + assert {item.value for item in samples} == set(CODES) + for item in samples: + assert item.frequency == pytest.approx(1 / 3, abs=0.02) + finally: + _teardown(postgres_backend) + + def test_limit_bounds_the_listing(self, postgres_backend): + _setup(postgres_backend) + try: + assert ( + len( + postgres_backend.introspector.sample_column_values( + TABLE, "code", limit=2 + ) + ) + == 2 + ) + finally: + _teardown(postgres_backend) + + def test_null_is_not_a_most_common_value(self, postgres_backend): + """PostgreSQL tracks NULLs in ``null_frac``, never in the value list. + + Worth pinning because the other implementation of this family does the + opposite: SQLite aggregates the column, so its listing includes a NULL + row. Same question, two engines, two shapes -- which is exactly why the + caller reads ``null_fraction`` rather than looking for a NULL entry. + """ + _setup(postgres_backend) + try: + samples = postgres_backend.introspector.sample_column_values( + TABLE, "note", limit=10 + ) + assert {item.value for item in samples} == {"x", "y"} + assert all(item.value is not None for item in samples) + finally: + _teardown(postgres_backend) From d32ea5168ff315aff8a753f3e0b4c46e99205556 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 3 Nov 2026 14:17:13 +0800 Subject: [PATCH 13/27] fix(postgres): read pg_stats arrays through array_to_json The new live test failed on the first real server, and the failure was the informative kind: `cannot subscript type anyarray`. `pg_stats` declares `histogram_bounds`, `most_common_vals` and `most_common_freqs` as `anyarray` -- a pseudo-type standing for "an array of whatever this column is" -- and PostgreSQL refuses both subscripting and `MIN(anyelement)` over a pseudo-type, since there is no element type to resolve an operator against. `array_to_json` is the one function declared over `anyarray` itself, so the arrays travel through it and the parser picks the endpoints out of the JSON. The cost, now stated where a reader will look: a bound arrives as a JSON scalar rather than the driver's own object. Numbers stay numbers; a date, a timestamp or a UUID reads as its ISO text. That is a limit of `pg_stats` rather than a choice -- nothing in the catalog can hand back a typed endpoint without knowing the column's type at plan time. Worth recording how this was found: the statement rendered perfectly, the roundtrip walk accepted it, and only executing it against a server exposed the pseudo-type. Rendering proves a statement is well-formed, not that it runs. --- .../postgres/introspection/introspector.py | 24 ++++-- .../impl/postgres/introspection/types.py | 6 ++ .../impl/postgres/mixins/introspection.py | 76 ++++++++----------- 3 files changed, 56 insertions(+), 50 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py index 40960060..998181b8 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py @@ -281,12 +281,11 @@ def _stats_list(value: Any) -> List[Any]: def _lower_of(current: Any, candidate: Any) -> Any: """The smaller of two statistics values, when they can be compared. - A histogram endpoint arrives in the column's own type (a ``date``, - a ``Decimal``) while a most-common value arrives as JSON (a string, - an ``int``). Comparing across those families would be misleading, so - only like kinds are reconciled: two numbers, or two strings -- the - latter covering text, and dates and uuids in their ISO spellings. - Everything else keeps the endpoint, and the docstring says so. + Both sides come out of ``to_jsonb``, so the shapes are JSON's: numbers, + strings, booleans and null. Only like kinds are reconciled -- two + numbers, or two strings, the latter covering text, and dates and uuids + in their ISO spellings -- because a JSON string and a JSON number have + no meaningful order between them. """ if current is None: return candidate @@ -350,6 +349,14 @@ def parse_column_data_profiles( was collected, and a ``pg_stat_reset()`` erases that time while leaving the statistics in place. Saying ``UNKNOWN`` keeps the two facts from being silently conflated. + + ``min_value``/``max_value`` are the histogram's endpoints merged with + the retained most-common values, because ANALYZE's histogram excludes + what it kept as common. Both arrive as JSON scalars: a number is a + number, while a date or a UUID is its text form. ``pg_stats`` is + declared over ``anyarray`` and cannot be subscripted or aggregated, so + a typed endpoint is not obtainable -- see the dialect's + ``_histogram_projection``. """ from .types import PostgresColumnDataProfile @@ -386,8 +393,11 @@ def parse_column_data_profiles( for value in self._stats_list(row.get("most_common_freqs")) if isinstance(value, (int, float)) ] + bounds = self._stats_list(row.get("histogram_bounds")) min_value, max_value = self._merged_bounds( - row.get("min_value"), row.get("max_value"), most_common + bounds[0] if bounds else None, + bounds[-1] if bounds else None, + most_common, ) collected_at = row.get("last_analyze") or row.get("last_autoanalyze") diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/types.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/types.py index d4c05f43..4775fc92 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/types.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/types.py @@ -67,6 +67,12 @@ class PostgresColumnDataProfile(ColumnDataProfile): ``min_value``/``max_value`` are the true extremes: the parser merges the histogram's endpoints with the most-common values, since ANALYZE's histogram excludes the values it kept as MCVs. + + They arrive as JSON scalars, not as the driver's own objects. ``pg_stats`` + is declared over ``anyarray``, a pseudo-type PostgreSQL refuses to + subscript or aggregate, so the array is read through ``to_jsonb``: a number + stays a number, while a date, a timestamp or a UUID reads as its ISO text. + Everything else on the profile is still native. """ #: Agreement between physical row order and the column's ordering. diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 16a1f03b..059ffa35 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -3167,42 +3167,32 @@ def _stats_source(self, table_name): ) return source - def _histogram_endpoints(self): - """``(min, max)`` as the first and last histogram bound. - - Element access rather than an aggregate over ``unnest``: subscripting - an array that is NULL yields NULL, so both expressions are NULL-safe - without a CASE, and -- unlike ``MIN(anyelement)`` -- they do not - require the column's type to have a btree opclass. A ``json`` column - has statistics with no histogram, and a ``MIN()`` formulation would - refuse to plan for it while this one simply returns NULL. - - ``histogram_bounds`` excludes the values kept as ``most_common_vals``, - so its endpoints are the extremes of the *non-MCV* values. The parser - merges the MCV list back in; keeping that reconciliation out of SQL is - what lets it compare values the catalog returned in their real types. + def _histogram_projection(self): + """``histogram_bounds`` as JSON, which is the only readable shape. + + ``pg_stats.histogram_bounds`` is declared ``anyarray`` -- a pseudo-type + standing for "an array of whatever this column is" -- and PostgreSQL + refuses to subscript it (``cannot subscript type anyarray``) as well as + ``MIN(anyelement)`` over it, since a type without a btree opclass has no + aggregate. ``array_to_json`` is declared over ``anyarray``, so that is + the one call that accepts it as it is: the array travels as JSON and + the parser takes its endpoints. + + The cost is that a bound arrives as a JSON scalar rather than the + driver's own object: numbers stay numbers, while a date, a timestamp or + a UUID reads as its text form. That is a limit of ``pg_stats`` rather + than a choice -- nothing in the catalog can hand back a typed endpoint + without knowing the column's type at plan time -- and it is stated + where a reader will look. """ - from rhosocial.activerecord.backend.expression.advanced_functions import ArrayOperation - from rhosocial.activerecord.backend.expression.functions.array import ( - ArrayLengthFunction, - ) + from rhosocial.activerecord.backend.expression import FunctionCall - bounds = self._catalog_column("histogram_bounds", "s") - first = ArrayOperation( - self, - "ACCESS", - base_expr=bounds, - index_expr=self._catalog_literal(1), - alias="min_value", - ) - last = ArrayOperation( + return FunctionCall( self, - "ACCESS", - base_expr=bounds, - index_expr=ArrayLengthFunction(self, bounds, 1), - alias="max_value", + "array_to_json", + self._catalog_column("histogram_bounds", "s"), + alias="histogram_bounds", ) - return first, last def format_column_data_profile_query_statement( self, expr: "ColumnDataProfileQueryStatement" @@ -3220,10 +3210,12 @@ def format_column_data_profile_query_statement( comparison between two columns, not a constant -- and the conversion belongs next to the row estimate it uses. - The most-common arrays travel as JSON. They are ``anyarray``: their - element type is the profiled column's own, so any cast to a common type - would either fail to plan or lose the value; ``to_jsonb`` yields typed - JSON scalars, which is the one shape that survives all of them. + The three arrays all travel as JSON, for the same reason: ``anyarray`` + is a pseudo-type whose element type is the profiled column's own, so no + common cast exists and no aggregate is defined over it. + ``array_to_json`` yields typed JSON scalars, which is the one shape that + survives every element type; see :meth:`_histogram_projection` for what + that costs min/max. A column with no statistics row simply is not in the answer, and a never-analysed table answers with no rows at all -- "not measured" as @@ -3247,25 +3239,23 @@ def format_column_data_profile_query_statement( self, "eq", self._catalog_column("relname", "c"), self._catalog_value(table_name) ), ] - min_value, max_value = self._histogram_endpoints() projections = [ self._catalog_column("attname", "s", alias="column_name"), self._catalog_column("schemaname", "s", alias="schema_name"), self._catalog_column("null_frac", "s", alias="null_fraction"), self._catalog_column("n_distinct", "s"), - min_value, - max_value, + self._histogram_projection(), self._catalog_column("reltuples", "c", alias="row_count"), self._catalog_column("correlation", "s"), FunctionCall( self, - "to_jsonb", + "array_to_json", self._catalog_column("most_common_vals", "s"), alias="most_common_vals", ), FunctionCall( self, - "to_jsonb", + "array_to_json", self._catalog_column("most_common_freqs", "s"), alias="most_common_freqs", ), @@ -3319,13 +3309,13 @@ def format_column_value_sample_query_statement( projections = [ FunctionCall( self, - "to_jsonb", + "array_to_json", self._catalog_column("most_common_vals", "s"), alias="most_common_vals", ), FunctionCall( self, - "to_jsonb", + "array_to_json", self._catalog_column("most_common_freqs", "s"), alias="most_common_freqs", ), From c9b4dc09f138184a4a059be56c47726cbc3c7c50 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 3 Nov 2026 14:26:15 +0800 Subject: [PATCH 14/27] fix(postgres): admit when the vintage of an estimate is not recorded The live test's second finding. `ANALYZE` ran, the figures were right, and `source` came back `unknown` -- because `pg_stat_all_tables.last_analyze` is written by the statistics collector *asynchronously*, so a profile read immediately after the analysis usually finds no timestamp yet. That is not a bug to paper over: it is the family's own rule working. The rule says an `ESTIMATE` carries the time it was collected, precisely so a stale number cannot pass for a fresh one. When the server has not recorded that time, the honest answer is that the age is unknown -- calling it `ESTIMATE` with an empty timestamp would claim an age the server never reported. So the pairing stays as it is and the reasoning is written down: two things leave the time unrecorded, a `pg_stat_reset()` and the collector's own lag, and neither is an error. The test now pins the invariant rather than one value -- `(source == "estimate") == (collected_at is not None)` -- which is the property that matters and holds whatever the collector happens to have flushed. --- .../impl/postgres/introspection/introspector.py | 9 ++++++--- .../backend/impl/postgres/mixins/introspection.py | 8 ++++---- .../backend/test_introspection_data_profile.py | 15 ++++++++++++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py index 998181b8..67e06226 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py @@ -346,9 +346,12 @@ def parse_column_data_profiles( ``source`` is ``ESTIMATE`` whenever the vintage is known and ``UNKNOWN`` when it is not: the figures are estimates either way, but the rule for this family is that an estimate must carry the time it - was collected, and a ``pg_stat_reset()`` erases that time while - leaving the statistics in place. Saying ``UNKNOWN`` keeps the two - facts from being silently conflated. + was collected. Two things can leave that time unrecorded, and neither + is an error: a ``pg_stat_reset()`` erases it while the statistics stay, + and the statistics collector records ``last_analyze`` asynchronously, + so a profile read immediately after ``ANALYZE`` usually finds no + timestamp yet. Saying ``UNKNOWN`` keeps a figure whose age is unknown + from claiming an age it does not have. ``min_value``/``max_value`` are the histogram's endpoints merged with the retained most-common values, because ANALYZE's histogram excludes diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 059ffa35..436578d4 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -249,10 +249,10 @@ def supports_column_data_profile(self) -> bool: and ``correlation`` are one row per column, so profiling a table costs one catalog query however large the table is. The price is that every figure is an *estimate* of the last ``ANALYZE``, which is why the - parser labels the family ``ESTIMATE`` and carries the analysis time - beside it. A table that was never analysed has no ``pg_stats`` row and - therefore no profile -- an honest empty answer rather than fabricated - zeros. + parser labels the family ``ESTIMATE`` when the analysis time is + recorded and ``UNKNOWN`` when it is not -- see its docstring. A table + that was never analysed has no ``pg_stats`` row and therefore no + profile: an honest empty answer rather than fabricated zeros. """ return True diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py index 987b63f5..3f82c60e 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py @@ -107,14 +107,23 @@ def test_profile_matches_a_hand_count(self, postgres_backend): _teardown(postgres_backend) def test_every_profile_carries_its_provenance(self, postgres_backend): - """An estimate must say it is one, and when it was taken.""" + """An estimate carries its vintage, and an unknown one admits it. + + The invariant is the pair, not one value: ``ESTIMATE`` means the age is + known and recorded, ``UNKNOWN`` means the figures are there but their + age is not. Right after ``ANALYZE`` the second is what answers -- the + statistics collector records ``last_analyze`` asynchronously, so the + timestamp is usually still empty when the profile is read. + """ _setup(postgres_backend) try: profiles = postgres_backend.introspector.list_column_data_profiles(TABLE) assert profiles for profile in profiles: - assert profile.source == "estimate" - assert profile.collected_at, "an estimate must carry its vintage" + assert profile.source in ("estimate", "unknown") + assert (profile.source == "estimate") == ( + profile.collected_at is not None + ), "ESTIMATE must carry a vintage, UNKNOWN must not claim one" assert profile.row_count == 1000 finally: _teardown(postgres_backend) From cab9e1ccff59be3be8e34f42b1d3b004eed4add3 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 3 Nov 2026 15:07:20 +0800 Subject: [PATCH 15/27] feat(postgres): measure the two-sided coverage, the family's last probe With this statement the data-profile family is complete on PostgreSQL. What it answers is the overlap of two columns' value domains, and it answers in one row carrying both shares: how much of the left column the right one accounts for, and the same the other way round. Neither figure means much alone -- a left share of 1.0 beside a right share of 0.2 says every tested value exists over there while most of that domain is unaccounted for -- so a single similarity score would average away exactly the asymmetry a caller needs to decide whether a join is worth writing. The semantics are the ones the SQLite implementation established, spelled for this engine. Both fractions count distinct values rather than rows, because a key repeated a thousand times that matches once would otherwise read as well covered. NULLs are excluded from the numerators and from the divisors: a NULL is not a value that could ever join, and in the numerator the exclusion is load-bearing, since SELECT DISTINCT would otherwise treat it as one value and credit the tested side with a match it does not have. Each divisor is guarded by NULLIF, so a column with nothing to match answers NULL rather than a zero that would read as a real measurement; the parser already turns that into "no measurement". The `* 1.0` is not decoration either: PostgreSQL truncates integer division, so without it every share below one would come back as zero. This is the one formatter here whose identifiers are not the catalog's. Both sides are user tables, so the source and its columns are built as Table and Column nodes and the dialect quotes them, schema and aliases included; _catalog_column writes names the way pg_catalog spells them, which is how a catalog relation is spelled and not how a user's table is. The two sides get distinct aliases so the measurement can name which direction it is reporting. The price is stated rather than hidden. supports_join_coverage now answers True, and its docstring and the formatter's say the same thing: this is a double scan plus an intersection, it belongs behind an explicit call and never in an automatic pass, and its fractions are a lower bound when the two columns' collations differ, because values the server would compare as equal may be counted as distinct here. The roundtrip table drops its last introspection row. That entry existed only because the statement did not, and it is the table's exactness assertion that made removing it part of writing the formatter. The data-profile test gains a coverage class measured against two hand-sized tables -- 1..4 against 2..6 must answer 3/4 and 3/5 -- together with the three cases that separate a measurement from a plausible guess: repeated values counted once, disjoint domains answering 0.0 rather than None, and a side with no non-NULL value answering no fraction at all. No PostgreSQL server is reachable from this machine, so CI is where the statement meets the engine for the first time. Its arithmetic was checked locally by executing the rendered text against SQLite under an attached `public` schema, which answered 0.75/0.6, 0.5/0.5, 0.0/0.0, NULL/0.0 and NULL/NULL for the cases the tests assert. --- .../impl/postgres/mixins/introspection.py | 195 +++++++++++++++++- .../backend/test_expression_roundtrip_all.py | 14 +- .../test_introspection_data_profile.py | 119 +++++++++++ 3 files changed, 305 insertions(+), 23 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py index 436578d4..e5da13c6 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/introspection.py @@ -37,6 +37,7 @@ ColumnInfoQueryStatement, IndexInfoQueryStatement, ForeignKeyQueryStatement, + JoinCoverageQueryStatement, ViewListQueryStatement, ViewInfoQueryStatement, TriggerListQueryStatement, @@ -269,17 +270,21 @@ def supports_column_value_sample(self) -> bool: def supports_join_coverage(self) -> bool: - """False: the two-sided statement is not written for PostgreSQL yet. - - The engine can measure the overlap -- scan both columns, intersect the - distinct domains -- so this is a missing statement rather than a - missing capability. Answering True would promise a formatter that does - not exist, which is precisely the "declared but not rendered" shape the - roundtrip walk rejects; the probe stays False until the statement is - written. Note that the measurement is a double scan, so it belongs - behind an explicit call rather than in any automatic pass. + """PostgreSQL measures overlap by counting distinct values on both sides. + + The engine can read its own rows, so the probe answers ``True``: what + is expensive is the *statement*, not the capability, and pricing the + question into a ``False`` would claim PostgreSQL cannot answer at all. + There is no catalogue shortcut here either -- ``pg_stats`` keeps a + bounded, sampled most-common list per column, so an overlap computed + from it would be a guess dressed as a measurement, while counting the + values is exact. + + That exactness is paid for: the measurement is a double scan plus an + intersection, which is why it belongs behind an explicit call and never + in any automatic pass. The statement's docstring states the rest. """ - return False + return True def get_supported_introspection_scopes(self) -> List[IntrospectionScope]: @@ -3324,5 +3329,175 @@ def format_column_value_sample_query_statement( projections, self._catalog_relation("pg_stats", "s"), conditions, None ) + def _user_relation(self, table_name: str, alias: str, schema: str): + """A user table as a FROM source, quoted by the dialect. + + The opposite of :meth:`_catalog_relation`: ``pg_catalog`` spells its + own names lower-case and never quoted, while a user table is whatever + its owner declared -- mixed case, a space, a reserved word -- so the + name, its schema and the alias are all left to the dialect's identifier + renderer. The alias is what tells the two sides of a coverage + measurement apart, and it is quoted for the same reason. + """ + from rhosocial.activerecord.backend.expression.objects import Table + from rhosocial.activerecord.backend.expression.sources import NamedRelationRef + + return NamedRelationRef( + self, + Table(self, table_name, schema_name=schema), + alias=alias, + ) + + def _coverage_fraction(self, column, source, other_column, other_source): + """One side of the overlap: the share of *column* the other side holds. + + The numerator counts the *distinct* values of *column* that also occur + in *other_column*, as ``COUNT(*)`` over a derived table of those + distinct values; the denominator counts *column*'s distinct non-NULL + values. Distinct on both sides is what makes the figure a coverage of + values rather than of rows -- a key repeated a thousand times that + matches once would otherwise read as well covered. + + ``IS NOT NULL`` is written on both halves rather than relied upon. In + the numerator it is load-bearing: ``SELECT DISTINCT`` treats a NULL as + one value, so counting it would credit the tested side with a match it + does not have. In the divisor ``COUNT(DISTINCT ...)`` skips NULLs on + its own, but the statement says so explicitly, because the share is of + the values that could join and that is the fact being measured. + ``NULLIF(..., 0)`` then makes a column with no value at all answer NULL + rather than 0, which the parser reads as "no fraction": the difference + between "nothing matched" and "nothing to match" survives to the + caller. + + ``* 1.0`` is load-bearing on this engine: PostgreSQL divides integers + by truncation, so ``COUNT(*) / COUNT(...)`` would answer 0 for every + share below one. + """ + from rhosocial.activerecord.backend.expression import ( + FunctionCall, + InPredicate, + Literal, + LogicalPredicate, + QueryStatement, + RowCountWildcard, + SelectModifier, + Subquery, + WhereClause, + ) + from rhosocial.activerecord.backend.expression.functions import CountFunction + from rhosocial.activerecord.backend.expression.sources import DerivedTableSource + + overlap = QueryStatement( + self, + select=[column], + from_=source, + where=WhereClause( + self, + LogicalPredicate( + self, + "AND", + column.is_not_null(), + InPredicate( + self, + column, + Subquery( + self, + QueryStatement( + self, + select=[other_column], + from_=other_source, + where=WhereClause(self, other_column.is_not_null()), + ), + ), + ), + ), + ), + select_modifier=SelectModifier.DISTINCT, + ) + covered = Subquery( + self, + QueryStatement( + self, + select=[CountFunction(self, RowCountWildcard(self))], + from_=DerivedTableSource(self, overlap, alias="covered"), + ), + ) + total = Subquery( + self, + QueryStatement( + self, + select=[ + FunctionCall( + self, + "NULLIF", + CountFunction(self, column, distinct=True), + Literal(self, 0, inline=True), + ) + ], + from_=source, + where=WhereClause(self, column.is_not_null()), + ), + ) + return (covered * Literal(self, 1.0, inline=True)) / total + + def format_join_coverage_query_statement( + self, expr: "JoinCoverageQueryStatement" + ) -> Tuple[str, tuple]: + """Format a two-sided measurement of two columns' value domains. + + Unlike every other statement in this module, this one names two user + tables rather than one catalogue relation: the question is a relation + between two objects, so both sides are carried explicitly and their + identifiers are the dialect's to quote (see :meth:`_user_relation`). + ``_catalog_column`` is deliberately not used here -- it writes names + the way ``pg_catalog`` spells them, which is not how a user's table is + spelled. + + The row carries both fractions because neither is meaningful alone. A + left share of 1.0 beside a right share of 0.2 says every tested value + exists on the other side while most of that other domain is + unaccounted for; a single similarity score would average the two and + describe neither. A caller deciding whether a join is worth writing + needs to know which direction is incomplete. + + Both figures are counted over *distinct* values, with NULLs excluded + from the numerators and the denominators alike -- a NULL is not a value + that could ever join, so it belongs in neither half of the fraction. + The denominators are ``NULLIF``-guarded so a column with nothing to + match answers NULL, which the parser reports as "no measurement" + instead of a zero overlap that would read as a real answer. See + :meth:`_coverage_fraction` for both rules in place. + + The cost is the two sides read for their distinct domains plus an + intersection of the two -- a double scan, and the reason the probe + belongs behind an explicit call rather than in any automatic pass. + + Collations are not reconciled. PostgreSQL compares the two columns + under whichever collation applies, so values that the database would + treat as equal may be counted as distinct here, and the fractions are + a *lower* bound across mismatched collations -- never an upper one. + """ + from rhosocial.activerecord.backend.expression import Column, QueryStatement + + params = expr.get_params() + schema = params.get("schema") or self.default_schema + + left_source = self._user_relation(params.get("left_table"), "l", schema) + right_source = self._user_relation(params.get("right_table"), "r", schema) + left_column = Column(self, params.get("left_column"), table="l") + right_column = Column(self, params.get("right_column"), table="r") + + return QueryStatement( + self, + select=[ + self._coverage_fraction( + left_column, left_source, right_column, right_source + ).as_("left_coverage"), + self._coverage_fraction( + right_column, right_source, left_column, left_source + ).as_("right_coverage"), + ], + ).to_sql() + __all__ = ["PostgresIntrospectionCapabilityMixin"] diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py index 640a1a75..31044a53 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py @@ -129,10 +129,7 @@ class name -- ``Literal`` the single exception, naming its fourteen value kinds import pytest -from rhosocial.activerecord.backend.dialect.exceptions import ( - ProtocolNotImplementedError, - UnsupportedFeatureError, -) +from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError from rhosocial.activerecord.backend.expression import graph as graph_mod from rhosocial.activerecord.backend.expression.advanced_functions import ( CaseExpression, @@ -1226,15 +1223,6 @@ def _no_such_formatter(formatter_name: str) -> str: return f"does not support the '{formatter_name}' statement" LEGITIMATE_NON_RENDERS = { - # ---- the one data-profile family this dialect does not answer ---- - # The profile and the value listing are now rendered (see - # ``format_column_data_profile_query_statement`` and its sample twin), so - # only the two-sided coverage measurement remains unclaimed: it is a - # double scan rather than a catalogue read, and the statement has not been - # written for PostgreSQL yet. - "rhosocial.activerecord.backend.expression.introspection.JoinCoverageQueryStatement": ( - ProtocolNotImplementedError, 'does not implement ', - ), # ---- a marker class that exists in order to refuse ------------------ # ``_PostgresUnsupportedDomainAction`` is constructed precisely so an ALTER DOMAIN # action PostgreSQL cannot express gets a message naming the action. The diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py index 3f82c60e..ff857d04 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_data_profile.py @@ -25,6 +25,12 @@ branch of the min/max merge. A thousand rows guarantees the histogram exists for the unique columns and the MCV list exists for the low-cardinality one, so both branches are exercised. + +Coverage is measured against two small tables instead. Unlike the profile it +reads no statistics at all -- it counts the rows -- so there is nothing to +``ANALYZE`` and nothing to sample, and hand-sized domains are exactly what +makes the expected fractions checkable by eye: left ``{1,2,3,4}`` against right +``{2,3,4,5,6}`` must answer three quarters and three fifths. """ import pytest @@ -188,3 +194,116 @@ def test_null_is_not_a_most_common_value(self, postgres_backend): assert all(item.value is not None for item in samples) finally: _teardown(postgres_backend) + + +LEFT_TABLE = "intro_coverage_left" +RIGHT_TABLE = "intro_coverage_right" + + +def _coverage_setup(backend, left_values, right_values): + """Two one-column tables, filled with the values the case was written for.""" + for table in (LEFT_TABLE, RIGHT_TABLE): + backend.execute(f"DROP TABLE IF EXISTS {table}") + backend.execute(f"CREATE TABLE {LEFT_TABLE} (code integer)") + backend.execute(f"CREATE TABLE {RIGHT_TABLE} (code integer)") + backend.execute_many( + f"INSERT INTO {LEFT_TABLE} (code) VALUES (%s)", + [(value,) for value in left_values], + ) + backend.execute_many( + f"INSERT INTO {RIGHT_TABLE} (code) VALUES (%s)", + [(value,) for value in right_values], + ) + + +def _coverage_teardown(backend): + for table in (LEFT_TABLE, RIGHT_TABLE): + backend.execute(f"DROP TABLE IF EXISTS {table}") + + +def _coverage(backend): + return backend.introspector.estimate_join_coverage( + LEFT_TABLE, "code", RIGHT_TABLE, "code" + ) + + +class TestJoinCoverage: + """How far two columns' distinct value domains overlap. + + Every expectation is arithmetic rather than a recorded run: ``left`` has + ``{1,2,3,4}`` and ``right`` has ``{2,3,4,5,6}``, so three of the four left + values are matched and three of the five right ones are. The two figures + are different on purpose -- an implementation that answered the same number + twice, or averaged them, would still look plausible. + """ + + def test_both_fractions_match_a_hand_count(self, postgres_backend): + _coverage_setup(postgres_backend, range(1, 5), range(2, 7)) + try: + coverage = _coverage(postgres_backend) + assert coverage is not None + assert coverage.left_coverage == pytest.approx(3 / 4) + assert coverage.right_coverage == pytest.approx(3 / 5) + finally: + _coverage_teardown(postgres_backend) + + def test_values_are_counted_not_rows(self, postgres_backend): + """A repeated key that matches once is one covered value, not three. + + The left table holds four rows but only two distinct values, and the + repeated value is the matched one: counting rows would answer 3/4 where + the share of values is 1/2. + """ + _coverage_setup(postgres_backend, (2, 2, 2, 1), (2, 3)) + try: + coverage = _coverage(postgres_backend) + assert coverage is not None + assert coverage.left_coverage == pytest.approx(1 / 2) + assert coverage.right_coverage == pytest.approx(1 / 2) + finally: + _coverage_teardown(postgres_backend) + + def test_no_overlap_is_zero_not_none(self, postgres_backend): + """Disjoint domains are a measurement: both sides really are uncovered. + + ``None`` means "the engine could not measure", which is a different + answer from "nothing matched", and a caller deciding whether a join is + worth writing has to be able to tell them apart. + """ + _coverage_setup(postgres_backend, (1, 2, 3), (10, 11)) + try: + coverage = _coverage(postgres_backend) + assert coverage is not None + assert coverage.left_coverage == 0.0 + assert coverage.right_coverage == 0.0 + finally: + _coverage_teardown(postgres_backend) + + def test_a_side_with_no_value_answers_no_fraction(self, postgres_backend): + """A column of nothing but NULLs has no share to report. + + NULLs are excluded from the divisors as well as the numerators, and + ``NULLIF`` turns the empty denominator into NULL. So the left side -- + which has no value at all -- answers ``None``, while the right side, + which has values that simply did not match, answers a real zero. + """ + _coverage_setup(postgres_backend, (None, None), (1, 2)) + try: + coverage = _coverage(postgres_backend) + assert coverage is not None + assert coverage.left_coverage is None + assert coverage.right_coverage == 0.0 + finally: + _coverage_teardown(postgres_backend) + + def test_neither_side_having_a_value_is_no_measurement(self, postgres_backend): + """When both fractions are NULL the parser reports nothing measured. + + That is what keeps an all-NULL pair from reading as "zero overlap": + there was no overlap to measure, and the answer says exactly that. + """ + _coverage_setup(postgres_backend, (None,), (None,)) + try: + assert _coverage(postgres_backend) is None + finally: + _coverage_teardown(postgres_backend) From b1652734e582b355240cc59eb316161a7fa20225 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Tue, 3 Nov 2026 16:05:39 +0800 Subject: [PATCH 16/27] feat(postgres): let the CLI ask every family the dialect answers The introspect mapping named eleven types while the dialect answered six operational and three data families that no type reached. A capability the CLI cannot reach is the same defect as a statement declared and never rendered: the dialect says the server in front of us can answer, and no question gets the answer out. Core's introspection rule states it in one line -- the type lands in the same change as the capability -- so the nine now ride _TYPE_CAPABILITIES beside the existing entries, through the same _require_capability gate, so a server whose probe says False is still refused rather than answered with an empty listing. unused-indexes reads list_unused_indexes, the indexes the statistics collector reports as unchosen since the statistics were last reset. table-stats reads the per-table usage counters and runtime reads the live measurements, both as the statistics collector keeps them, and index-stats answers "how much" per index where unused-indexes answers "ever". All four are listings over the connection's scope, so none takes a name. dependencies asks list_object_dependencies: the whole graph without a name, or one object's dependants with it, which is the "what breaks if I drop this" question. ddl asks get_ddl_statement for one named table and prints the statement itself in table output, because a CREATE is text and folding it into a one-column table buries it; the structured formats keep the whole row with object_type and native beside it. The three data families ask what the rows hold rather than what the schema declares. column-profiles asks list_column_data_profiles for every column of one table, value-samples asks sample_column_values for one column of it, and join-coverage asks estimate_join_coverage for both sides of a two-column measurement -- table and column on each side, four names in all. The column is named on --column and the right-hand side on --right-table and --right-column; a flag the chosen type has no question for is refused rather than ignored, the same way a name already was, and a measurement that comes back empty is reported as no measurement instead of a row of nulls that would read as zero overlap. Both dispatch chains carry the nine branches. The sync and async handlers are written out separately because the executor is the only thing that differs, and the tests drive both: one case per type asserts the method and every argument, and the refusal path is checked across the whole type list. The offline CLI tests' fake introspector gains one method per family, so the parametrized gates now cover every advertised type instead of the first eleven. Only the paths that need a live catalogue are left to CI; on this machine there is no PostgreSQL, so the answers are pinned at the dispatch boundary rather than read from a running server. --- .../backend/impl/postgres/cli/introspect.py | 357 +++++++++++++++++- .../test_introspection_cli_capabilities.py | 230 ++++++++++- 2 files changed, 579 insertions(+), 8 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py index 784dd5ad..ba1a16de 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py @@ -3,7 +3,11 @@ PostgreSQL introspect includes the 'extensions' type in addition to the standard types, plus the two schema-object families this backend -answers for on its own: materialized views and sequences. +answers for on its own: materialized views and sequences. The families the +catalog and the statistics collector answer beside the schema structure -- +unused indexes, usage counters, runtime metrics, dependencies, DDL and the +data profiles -- are offered too: the dialect declares it can answer them, +so the CLI has to be able to ask. A type either lists a family or, when it names a single catalogue object, answers for that one object; ``NAME`` is required, optional or refused @@ -35,9 +39,11 @@ #: than run and answered with an empty list: "this server cannot report #: sequences" and "this database has no sequences" are different answers. #: -#: One entry per family the catalog reports independently. Partition metadata -#: has a switch too, but no introspector question of its own -- it is read as -#: part of the DDL a partitioned table carries -- so it is not a type here. +#: One entry per family this backend answers independently, whether the answer +#: comes from the catalog, from the statistics collector or from the rows +#: themselves. Partition metadata has a switch too, but no introspector +#: question of its own -- it is read as part of the DDL a partitioned table +#: carries -- so it is not a type here. _TYPE_CAPABILITIES = { "tables": "supports_table_introspection", "views": "supports_view_introspection", @@ -53,6 +59,24 @@ # figures ride the listing's projection, and this type asks it for one # name rather than making the caller read the whole listing. "table-storage": "supports_table_storage_introspection", + # The operational families, read from the statistics collector rather + # than the catalog: how the server is behaving, not what it contains. + # They are listings over the connection's scope, so none takes a name -- + # ``dependencies`` may be narrowed to one object's edges but answers the + # whole graph without one. + "unused-indexes": "supports_unused_indexes_detection", + "table-stats": "supports_table_stats", + "runtime": "supports_runtime_stats", + "index-stats": "supports_index_stats", + "dependencies": "supports_object_dependencies", + # The one-object families: the DDL that recreates a named table, and the + # data profiles of one table or one column of it. They read the catalog, + # pg_stats or the rows themselves, and every answer carries how it was + # produced so the caller need not assume which. + "ddl": "supports_ddl_extraction", + "column-profiles": "supports_column_data_profile", + "value-samples": "supports_column_value_sample", + "join-coverage": "supports_join_coverage", } INTROSPECT_TYPES = list(_TYPE_CAPABILITIES) @@ -95,6 +119,54 @@ def _refuse_name(args: argparse.Namespace, hint: str) -> None: sys.exit(2) +def _refuse_unused_arguments(args: argparse.Namespace) -> None: + """Refuse a flag the chosen type has no question for. + + The same defect as a silently dropped name, one flag over: ``--column`` + beside ``introspect tables`` would be ignored and the listing printed as + if the column had been part of the question. Only the families whose + probe takes a column read the flag, so only they may be given it. + + The flags are read with ``getattr`` because the handler is also driven + with hand-built namespaces (the offline tests), which carry no flags at + all -- an absent attribute is "not given", not an error. + """ + column = getattr(args, "column", None) + right_table = getattr(args, "right_table", None) + right_column = getattr(args, "right_column", None) + + if column is not None and args.type not in ("value-samples", "join-coverage"): + print( + f"Error: '{args.type}' does not take --column; " + "use 'value-samples' or 'join-coverage'", + file=sys.stderr, + ) + sys.exit(2) + if (right_table is not None or right_column is not None) and args.type != "join-coverage": + print( + f"Error: '{args.type}' does not take --right-table or --right-column; " + "use 'join-coverage'", + file=sys.stderr, + ) + sys.exit(2) + + +def _display_ddl(provider, statement, args) -> None: + """Show a DDL statement in the form that stays readable. + + A ``CREATE TABLE`` is text: a one-row table folds it into a cell that + wraps at the terminal width and interleaves with the other columns. Table + output therefore prints the statement itself, while the structured + formats keep the whole row so JSON and CSV still carry ``object_type`` + and ``native`` beside the text. + """ + if args.output == "table": + print(statement.ddl or "") + return + data = _serialize_for_output([statement]) + provider.display_results(data, title=f"DDL of {statement.object_name}") + + def create_parser(subparsers): """Create the introspect subcommand parser.""" parser = subparsers.add_parser( @@ -139,6 +211,29 @@ def create_parser(subparsers): # Get one table's storage split (total, data, index bytes; row estimate) %(prog)s table-storage orders --database mydb + # Indexes no query has used since the statistics were reset, and the + # per-index counters that say how much the rest are used + %(prog)s unused-indexes --database mydb + %(prog)s index-stats --database mydb + + # Per-table usage counters and live runtime measurements + %(prog)s table-stats --database mydb + %(prog)s runtime --database mydb + + # The dependency graph, or one object's dependants + %(prog)s dependencies --database mydb + %(prog)s dependencies orders --database mydb + + # The DDL that recreates one table + %(prog)s ddl orders --database mydb + + # Profile every column of a table from its data, and sample one column + %(prog)s column-profiles orders --database mydb + %(prog)s value-samples orders --column status --database mydb + + # Measure how two columns' value domains overlap + %(prog)s join-coverage orders --column buyer_id --right-table users --right-column id + # Output as JSON %(prog)s tables --database mydb -o json @@ -180,12 +275,33 @@ def create_parser(subparsers): "name", nargs="?", default=None, - help="Object name. Required for columns, indexes, foreign-keys " - "(the table they belong to) and table-storage (the table itself); " - "optional for triggers (a table) " + help="Object name. Required for columns, indexes, foreign-keys, " + "table-storage, ddl and column-profiles (the table they belong to), " + "and for value-samples and join-coverage (the left table); " + "optional for triggers (a table), dependencies (one object) " "and for views, materialized-views, sequences and extensions " "(one object); refused for tables and database", ) + # The three families that ask about a column or a second table carry the + # names the probe needs. They stay flags rather than more positionals + # because ``name`` already means "the table" everywhere else, and + # ``value-samples orders.email`` would read as a dotted object name. + parser.add_argument( + "--column", + default=None, + help="Column name: the column to sample for value-samples, and the " + "left column for join-coverage", + ) + parser.add_argument( + "--right-table", + default=None, + help="Right-hand table for join-coverage", + ) + parser.add_argument( + "--right-column", + default=None, + help="Right-hand column for join-coverage", + ) parser.add_argument( "--include-system", action="store_true", @@ -274,6 +390,7 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): # question was read. backend.introspect_and_adapt() _require_capability(backend, args) + _refuse_unused_arguments(args) introspector = backend.introspector @@ -406,6 +523,124 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): data = _serialize_for_output([info]) provider.display_results(data, title=f"Table storage {args.name}") + # ---- the operational families ------------------------------------- + # These read the statistics collector, not the catalog: each one is a + # listing of the whole connection's scope, so none takes a name (the + # dependency graph may be narrowed to one object, and answers the + # whole graph without it). + + elif args.type == "unused-indexes": + indexes = introspector.list_unused_indexes(schema=None) + data = _serialize_for_output(indexes) + provider.display_results(data, title="Unused Indexes") + + elif args.type == "table-stats": + stats = introspector.list_table_usage_stats(schema=None) + data = _serialize_for_output(stats) + provider.display_results(data, title="Table Usage Stats") + + elif args.type == "runtime": + metrics = introspector.list_runtime_metrics(schema=None) + data = _serialize_for_output(metrics) + provider.display_results(data, title="Runtime Metrics") + + elif args.type == "index-stats": + stats = introspector.list_index_usage_stats(schema=None) + data = _serialize_for_output(stats) + provider.display_results(data, title="Index Usage Stats") + + elif args.type == "dependencies": + # A name narrows the graph to the edges that object carries -- + # "what breaks if I drop this" -- rather than asking a different + # family, so it is optional here the way a trigger's table is. + dependencies = introspector.list_object_dependencies(object_name=args.name, schema=None) + data = _serialize_for_output(dependencies) + title = f"Dependencies of {args.name}" if args.name else "Object Dependencies" + provider.display_results(data, title=title) + + # ---- the one-object families -------------------------------------- + # The DDL that recreates a table, and the profiles of one table or one + # column of it: the name is the table and the column is named on its + # own flag. + + elif args.type == "ddl": + if not args.name: + print("Error: Table name is required for 'ddl' introspection", file=sys.stderr) + sys.exit(1) + statement = introspector.get_ddl_statement(args.name, object_type=None, schema=None) + if statement is None: + # Not "no such table": the object exists, and PostgreSQL + # answers with no text rather than an assembled fragment that + # would read as authoritative. The two are worth telling + # apart, so this exits without printing an empty row. + print(f"Error: no DDL available for '{args.name}'", file=sys.stderr) + sys.exit(1) + _display_ddl(provider, statement, args) + + elif args.type == "column-profiles": + if not args.name: + print( + "Error: Table name is required for 'column-profiles' introspection", + file=sys.stderr, + ) + sys.exit(1) + profiles = introspector.list_column_data_profiles(args.name, schema=None) + data = _serialize_for_output(profiles) + provider.display_results(data, title=f"Column profiles of {args.name}") + + elif args.type == "value-samples": + if not args.name: + print( + "Error: Table name is required for 'value-samples' introspection", + file=sys.stderr, + ) + sys.exit(1) + if not args.column: + print( + "Error: --column is required for 'value-samples' introspection", + file=sys.stderr, + ) + sys.exit(1) + samples = introspector.sample_column_values(args.name, args.column, schema=None) + data = _serialize_for_output(samples) + provider.display_results(data, title=f"Values of {args.name}.{args.column}") + + elif args.type == "join-coverage": + if not args.name: + print( + "Error: Left table name is required for 'join-coverage' introspection", + file=sys.stderr, + ) + sys.exit(1) + if not args.column: + print( + "Error: --column is required for 'join-coverage' introspection", + file=sys.stderr, + ) + sys.exit(1) + if not args.right_table or not args.right_column: + print( + "Error: --right-table and --right-column are required for " + "'join-coverage' introspection", + file=sys.stderr, + ) + sys.exit(1) + coverage = introspector.estimate_join_coverage( + args.name, args.column, args.right_table, args.right_column, schema=None + ) + if coverage is None: + # A column with nothing to match answers "no measurement", + # which is not zero overlap; printing a row of nulls would + # read as a measured zero. + print( + f"Error: no coverage could be measured for {args.name}.{args.column} " + f"and {args.right_table}.{args.right_column}", + file=sys.stderr, + ) + sys.exit(1) + data = _serialize_for_output([coverage]) + provider.display_results(data, title="Join Coverage") + except ConnectionError as e: provider.display_connection_error(e) sys.exit(1) @@ -428,6 +663,7 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider # before a capability switch -- or a formatter -- can answer. await backend.introspect_and_adapt() _require_capability(backend, args) + _refuse_unused_arguments(args) introspector = backend.introspector @@ -559,6 +795,113 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider data = _serialize_for_output([info]) provider.display_results(data, title=f"Table storage {args.name}") + # The async twins of the nine branches above: same method, same + # argument and same title, awaited. They are written out rather than + # shared with the sync chain because the executor is the only thing + # that differs, and a shared body would have to carry an ``await`` + # flag that reads worse than the duplicate. + + elif args.type == "unused-indexes": + indexes = await introspector.list_unused_indexes(schema=None) + data = _serialize_for_output(indexes) + provider.display_results(data, title="Unused Indexes") + + elif args.type == "table-stats": + stats = await introspector.list_table_usage_stats(schema=None) + data = _serialize_for_output(stats) + provider.display_results(data, title="Table Usage Stats") + + elif args.type == "runtime": + metrics = await introspector.list_runtime_metrics(schema=None) + data = _serialize_for_output(metrics) + provider.display_results(data, title="Runtime Metrics") + + elif args.type == "index-stats": + stats = await introspector.list_index_usage_stats(schema=None) + data = _serialize_for_output(stats) + provider.display_results(data, title="Index Usage Stats") + + elif args.type == "dependencies": + dependencies = await introspector.list_object_dependencies( + object_name=args.name, schema=None + ) + data = _serialize_for_output(dependencies) + title = f"Dependencies of {args.name}" if args.name else "Object Dependencies" + provider.display_results(data, title=title) + + elif args.type == "ddl": + if not args.name: + print("Error: Table name is required for 'ddl' introspection", file=sys.stderr) + sys.exit(1) + statement = await introspector.get_ddl_statement( + args.name, object_type=None, schema=None + ) + if statement is None: + print(f"Error: no DDL available for '{args.name}'", file=sys.stderr) + sys.exit(1) + _display_ddl(provider, statement, args) + + elif args.type == "column-profiles": + if not args.name: + print( + "Error: Table name is required for 'column-profiles' introspection", + file=sys.stderr, + ) + sys.exit(1) + profiles = await introspector.list_column_data_profiles(args.name, schema=None) + data = _serialize_for_output(profiles) + provider.display_results(data, title=f"Column profiles of {args.name}") + + elif args.type == "value-samples": + if not args.name: + print( + "Error: Table name is required for 'value-samples' introspection", + file=sys.stderr, + ) + sys.exit(1) + if not args.column: + print( + "Error: --column is required for 'value-samples' introspection", + file=sys.stderr, + ) + sys.exit(1) + samples = await introspector.sample_column_values(args.name, args.column, schema=None) + data = _serialize_for_output(samples) + provider.display_results(data, title=f"Values of {args.name}.{args.column}") + + elif args.type == "join-coverage": + if not args.name: + print( + "Error: Left table name is required for 'join-coverage' introspection", + file=sys.stderr, + ) + sys.exit(1) + if not args.column: + print( + "Error: --column is required for 'join-coverage' introspection", + file=sys.stderr, + ) + sys.exit(1) + if not args.right_table or not args.right_column: + print( + "Error: --right-table and --right-column are required for " + "'join-coverage' introspection", + file=sys.stderr, + ) + sys.exit(1) + coverage = await introspector.estimate_join_coverage( + args.name, args.column, args.right_table, args.right_column, schema=None + ) + if coverage is None: + print( + f"Error: no coverage could be measured for {args.name}.{args.column} " + f"and {args.right_table}.{args.right_column}", + file=sys.stderr, + ) + sys.exit(1) + data = _serialize_for_output([coverage]) + provider.display_results(data, title="Join Coverage") + except ConnectionError as e: provider.display_connection_error(e) sys.exit(1) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_capabilities.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_capabilities.py index fd06f26c..adc0a143 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_capabilities.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_capabilities.py @@ -19,6 +19,7 @@ from rhosocial.activerecord.backend.impl.postgres.cli import introspect as introspect_mod from rhosocial.activerecord.backend.impl.postgres.cli.introspect import INTROSPECT_TYPES from rhosocial.activerecord.backend.impl.postgres.dialect import PostgresDialect +from rhosocial.activerecord.backend.introspection.types import DDLStatement, JoinCoverage def parse_introspect_args(argv): @@ -57,7 +58,7 @@ def __getattr__(self, item): try: return self[item] except KeyError: - raise AttributeError(item) + raise AttributeError(item) from None class FakeIntrospector: @@ -112,6 +113,71 @@ def list_sequences(self, schema=None): self._record("list_sequences", schema) return [{"name": "order_id_seq"}] + # The operational families: each answers a listing of its own, from the + # statistics collector rather than the catalog. + + def list_unused_indexes(self, schema=None): + self._record("list_unused_indexes", schema) + return [{"schema": "public", "table_name": "users", "index_name": "users_email_idx"}] + + def list_table_usage_stats(self, schema=None): + self._record("list_table_usage_stats", schema) + return [{"table_name": "users", "sequential_scans": 4}] + + def list_runtime_metrics(self, schema=None): + self._record("list_runtime_metrics", schema) + return [{"name": "active_sessions", "value": 3}] + + def list_index_usage_stats(self, schema=None): + self._record("list_index_usage_stats", schema) + return [{"table_name": "users", "index_name": "users_pkey", "scans": 9}] + + def list_object_dependencies(self, object_name=None, schema=None): + self._record("list_object_dependencies", object_name, schema) + return [{"dependent": "users_v", "referenced": "users"}] + + def get_ddl_statement(self, object_name, object_type=None, schema=None): + self._record("get_ddl_statement", object_name, object_type, schema) + return DDLStatement( + schema="public", + object_type="r", + object_name=object_name, + ddl=f"CREATE TABLE {object_name} (id integer);", + native=False, + ) + + # The data families: one table, or one column of it. + + def list_column_data_profiles(self, table_name, schema=None): + self._record("list_column_data_profiles", table_name, schema) + return [{"table_name": table_name, "column_name": "id"}] + + def sample_column_values(self, table_name, column_name, schema=None): + self._record("sample_column_values", table_name, column_name, schema) + return [{"value": "active", "count": 7}] + + def estimate_join_coverage( + self, left_table, left_column, right_table, right_column, schema=None + ): + self._record( + "estimate_join_coverage", + left_table, + left_column, + right_table, + right_column, + schema, + ) + return JoinCoverage( + left_schema="public", + left_table=left_table, + left_column=left_column, + right_schema="public", + right_table=right_table, + right_column=right_column, + left_coverage=1.0, + right_coverage=0.5, + ) + class FakeBackend: last_instance = None @@ -190,14 +256,37 @@ async def disconnect(self): # listing and picks the row -- one read, not the four a per-table info # answer would spend. "table-storage": "list_tables", + "unused-indexes": "list_unused_indexes", + "table-stats": "list_table_usage_stats", + "runtime": "list_runtime_metrics", + "index-stats": "list_index_usage_stats", + "dependencies": "list_object_dependencies", + "ddl": "get_ddl_statement", + "column-profiles": "list_column_data_profiles", + "value-samples": "sample_column_values", + "join-coverage": "estimate_join_coverage", } #: The name each type needs to reach its branch; the family listings take none. +#: ``dependencies`` accepts a name but answers the whole graph without one, so +#: the generic gate runs it unnamed and a case below pins the narrowed call. _ARGV = { "columns": ["users"], "indexes": ["users"], "foreign-keys": ["users"], "table-storage": ["users"], + "ddl": ["users"], + "column-profiles": ["users"], + "value-samples": ["users", "--column", "status"], + "join-coverage": [ + "users", + "--column", + "id", + "--right-table", + "orders", + "--right-column", + "user_id", + ], } @@ -318,6 +407,145 @@ def test_async(self, run_cli, type_name): assert FakeBackend.last_instance.disconnect_calls == 1 +class TestTheFamiliesTheDialectAnswers: + """One case per newly offered type: the method, and every argument with it. + + The parametrized gates above only prove that *some* introspector call is + made. These pin which one and with what, because a branch that reached + the right family with the wrong column would still pass a first-call + check -- and the columns are the whole question for the data families. + """ + + def _calls(self, run_cli, argv): + _, err, exc = run_cli(["introspect", *argv, "--database", "mydb", "-o", "json"]) + assert exc is None, f"stderr: {err}" + return FakeBackend.last_instance.introspector.calls + + def test_unused_indexes_reads_the_statistics_collector(self, run_cli): + assert self._calls(run_cli, ["unused-indexes"]) == [("list_unused_indexes", None)] + + def test_table_stats_reads_the_usage_counters(self, run_cli): + assert self._calls(run_cli, ["table-stats"]) == [("list_table_usage_stats", None)] + + def test_runtime_reads_the_live_metrics(self, run_cli): + assert self._calls(run_cli, ["runtime"]) == [("list_runtime_metrics", None)] + + def test_index_stats_reads_the_per_index_counters(self, run_cli): + assert self._calls(run_cli, ["index-stats"]) == [("list_index_usage_stats", None)] + + def test_dependencies_answers_the_whole_graph_unnamed(self, run_cli): + assert self._calls(run_cli, ["dependencies"]) == [("list_object_dependencies", None, None)] + + def test_dependencies_narrows_to_one_object_by_name(self, run_cli): + assert self._calls(run_cli, ["dependencies", "orders"]) == [ + ("list_object_dependencies", "orders", None) + ] + + def test_ddl_asks_for_the_named_table(self, run_cli): + assert self._calls(run_cli, ["ddl", "users"]) == [ + ("get_ddl_statement", "users", None, None) + ] + + def test_ddl_prints_the_statement_in_table_mode(self, run_cli): + """Table output is the SQL itself, not a one-cell table around it.""" + out, err, exc = run_cli(["introspect", "ddl", "users", "--database", "mydb"]) + assert exc is None, f"stderr: {err}" + assert "CREATE TABLE users" in out + + def test_column_profiles_asks_for_the_named_table(self, run_cli): + assert self._calls(run_cli, ["column-profiles", "users"]) == [ + ("list_column_data_profiles", "users", None) + ] + + def test_value_samples_names_the_table_and_the_column(self, run_cli): + assert self._calls(run_cli, ["value-samples", "users", "--column", "status"]) == [ + ("sample_column_values", "users", "status", None) + ] + + def test_join_coverage_passes_both_sides_of_the_measurement(self, run_cli): + assert self._calls( + run_cli, + [ + "join-coverage", + "orders", + "--column", + "customer_ref", + "--right-table", + "customers", + "--right-column", + "code", + ], + ) == [ + ( + "estimate_join_coverage", + "orders", + "customer_ref", + "customers", + "code", + None, + ) + ] + + +class TestTheNewArgumentsAreRequiredOrRefused: + """A missing name exits 1; a flag the type has no question for exits 2. + + Both are the request's fault rather than the run's, and both keep the + catalogue untouched: an argument that is neither required nor refused is + the silently-dropped one this handler keeps out. + """ + + def test_value_samples_without_a_column_exits_one(self, run_cli): + _, err, exc = run_cli(["introspect", "value-samples", "users", "--database", "mydb"]) + assert exc is not None and exc.code == 1 + assert "--column is required" in err + assert FakeBackend.last_instance.introspector.calls == [] + + def test_join_coverage_without_the_right_side_exits_one(self, run_cli): + _, err, exc = run_cli( + [ + "introspect", + "join-coverage", + "orders", + "--column", + "customer_ref", + "--database", + "mydb", + ] + ) + assert exc is not None and exc.code == 1 + assert "--right-table and --right-column are required" in err + assert FakeBackend.last_instance.introspector.calls == [] + + def test_ddl_without_a_table_name_exits_one(self, run_cli): + _, err, exc = run_cli(["introspect", "ddl", "--database", "mydb"]) + assert exc is not None and exc.code == 1 + assert "Table name is required for 'ddl'" in err + assert FakeBackend.last_instance.introspector.calls == [] + + def test_column_on_a_type_with_no_column_exits_two(self, run_cli): + _, err, exc = run_cli(["introspect", "tables", "--column", "status", "--database", "mydb"]) + assert exc is not None and exc.code == 2 + assert "'tables' does not take --column" in err + assert FakeBackend.last_instance.introspector.calls == [] + + def test_right_side_on_a_one_table_type_exits_two(self, run_cli): + _, err, exc = run_cli( + [ + "introspect", + "column-profiles", + "users", + "--right-table", + "orders", + "--database", + "mydb", + ] + ) + assert exc is not None and exc.code == 2 + assert "does not take --right-table or --right-column" in err + assert FakeBackend.last_instance.introspector.calls == [] + + class TestCapabilityGate: """A switch that answers False refuses the type instead of the listing.""" From 67744a4599b26f02807a15e3813f7485ad0caf7b Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Thu, 5 Nov 2026 12:56:53 +0800 Subject: [PATCH 17/27] test(ci): install the testsuite from the release line, not from a feature branch ix/column-constraint-gates has been merged: its tree and elease/v1.0.0.dev17 are identical (the branch carries 21 commits the release commit squashes into one, and the content matches). Pinning CI to the feature branch was always a temporary arrangement -- it meant every job installed a ref that only existed to get one fix in, and the day that branch was deleted the suite would have started failing to install rather than failing a test. The release line is now the thing to test against, which is also what the core repository already pins. --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76230519..b6fbc38b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,7 +69,7 @@ jobs: - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | @@ -284,7 +284,7 @@ jobs: - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | @@ -396,7 +396,7 @@ jobs: - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | @@ -494,7 +494,7 @@ jobs: - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@fix/column-constraint-gates" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Create postgres scenario config run: | From 04b078ee5113e776352d7eefc598f5dfe82da755 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Fri, 6 Nov 2026 15:34:24 +0800 Subject: [PATCH 18/27] fix(roundtrip): answer for the set-operation node core now ships Core now ships SetOperationStatement in backend.expression.statements, re-exported from backend.expression, with format_set_operation_statement on SetOperationMixin and a SetOperationSupport protocol. The roundtrip matrix enumerates every class in the package, so this class had to answer for itself in four parts: it renders and the rendered SQL plus its bind parameters are byte-identical after all three round-trips, or it refuses with UnsupportedFeatureError because the dialect lacks the feature, or it refuses with NotImplementedError because it is an abstract base, or it refuses for some other reason and is named in the not-rendered table with that reason. Every one of those is asserted exactly in both directions, so a class nobody listed is a failure and so is a row left behind after the class started working. Each backend here now gives the answer its own dialect earns, and pins the SQL where it renders, because a render verdict without the bytes says nothing about the dialect. --- .../backend/test_expression_roundtrip_all.py | 96 ++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py index 31044a53..47e1f618 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_expression_roundtrip_all.py @@ -162,7 +162,10 @@ class name -- ``Literal`` the single exception, naming its fourteen value kinds ) from rhosocial.activerecord.backend.expression.operators import ArithmeticOperation from rhosocial.activerecord.backend.expression.query_sources import SetOperation -from rhosocial.activerecord.backend.expression.statements.dql import QueryStatement +from rhosocial.activerecord.backend.expression.statements.dql import ( + QueryStatement, + SetOperationStatement, +) from rhosocial.activerecord.backend.expression.objects import ( Database, EdgeTable as EdgeTableObject, @@ -453,6 +456,20 @@ def _one_column_query(dialect, table="t"): ) +def _compound_arms(dialect): + """Two single-column SELECTs over two tables, no bind parameters.""" + return [ + QueryStatement( + dialect, select=[Column(dialect, "a")], from_=_table_obj(dialect) + ), + QueryStatement( + dialect, + select=[Column(dialect, "a")], + from_=_table_obj(dialect, "other"), + ), + ] + + def _integer_column(dialect, name="col"): """A column definition carrying a *dialect-bound* type. @@ -544,6 +561,16 @@ def register_specials(): right=_one_column_query(d, "other"), ), ) + # The statement half of the same pair. Same reason for needing a factory -- + # ``op`` is normalised through the word-form enum at construction and the + # guess's bare string is refused -- and the same reason for wanting one: + # PostgreSQL spells a compound SELECT, so the matrix can assert it. + register_special_constructor( + "statements.dql.SetOperationStatement", + lambda d: SetOperationStatement( + d, op="union_all", parts=_compound_arms(d) + ), + ) # -- the FROM side ------------------------------------------------------ register_special_constructor( @@ -2032,3 +2059,70 @@ def test_coverage_report(self, postgres_dialect): print(f" not constructible: {fqn}") for feature, methods in sorted(UNMODELLED_FORMATTERS.items()): print(f" not modelled: {feature} ({len(methods)} formatters)") + +# --------------------------------------------------------------------------- +# The set-operation statement, rendered +# --------------------------------------------------------------------------- +# +# The matrix says ``SetOperationStatement`` renders on PostgreSQL, which is a +# verdict about the dialect and not a claim about the bytes. This is where the +# bytes live: a compound SELECT over two real query parts, with the bind +# parameters asserted positionally so an ordering bug cannot hide behind valid +# SQL. The keyword is a second axis -- the plain ``UNION`` over the same arms +# must render, and must spell something different. + + +def test_set_operation_statement_renders_the_pinned_sql(postgres_dialect): + """``UNION ALL`` over two arms renders as the parts, joined by the keyword.""" + node = SetOperationStatement( + postgres_dialect, op="union_all", parts=_compound_arms(postgres_dialect) + ) + + rendered, params = node.to_sql() + + assert rendered == 'SELECT "a" FROM "t" UNION ALL SELECT "a" FROM "other"' + assert params == () + + +def test_set_operation_statement_spells_the_bare_union(postgres_dialect): + """The word form alone decides the keyword, so ``union`` differs.""" + node = SetOperationStatement( + postgres_dialect, op="union", parts=_compound_arms(postgres_dialect) + ) + + rendered, params = node.to_sql() + + assert rendered == 'SELECT "a" FROM "t" UNION SELECT "a" FROM "other"' + assert params == () + + +def test_set_operation_statement_parameters_concatenate_in_part_order( + postgres_dialect, +): + """Each arm's parameters arrive in the order the operator joins them.""" + dialect = postgres_dialect + parts = [ + QueryStatement( + dialect, + select=[Column(dialect, "a"), Literal(dialect, 1)], + from_=_table_obj(dialect), + where=ComparisonPredicate( + dialect, "=", Column(dialect, "a"), Literal(dialect, 5) + ), + ), + QueryStatement( + dialect, + select=[Column(dialect, "a"), Literal(dialect, 2)], + from_=_table_obj(dialect, "u"), + ), + ] + node = SetOperationStatement(dialect, op="union_all", parts=parts) + + rendered, params = node.to_sql() + + assert rendered == ( + 'SELECT "a", %s FROM "t" WHERE "a" = %s ' + "UNION ALL " + 'SELECT "a", %s FROM "u"' + ) + assert params == (1, 5, 2) From 28a2475efc833f07fc5ef463b7d0bca27e1b28ac Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sat, 7 Nov 2026 15:33:19 +0800 Subject: [PATCH 19/27] test(postgres): cite measured servers by scenario, not by address Provenance notes in src docstrings, test modules and docs examples now refer to the originating servers through scenario names instead of development-endpoint details; named-connection fixtures use neutral values. The scenario-to-endpoint mapping is defined only in the gitignored tests/config. No behaviour change; keeps dev-environment endpoints out of tracked source. --- docs/examples/chapter_04_ddl/ddl.py | 4 ++-- .../chapter_08_scenarios/parallel_workers/README.md | 2 +- .../backend/impl/postgres/mixins/ddl/constraint.py | 8 ++++---- .../backend/named_connection/example_connections.py | 12 ++++++------ .../backend/named_connection/test_resolver.py | 4 ++-- .../backend/postgres/test_ddl_alter_config.py | 2 +- .../test_postgres_constraint_capabilities.py | 8 ++++---- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/examples/chapter_04_ddl/ddl.py b/docs/examples/chapter_04_ddl/ddl.py index 26c45fca..fa986934 100644 --- a/docs/examples/chapter_04_ddl/ddl.py +++ b/docs/examples/chapter_04_ddl/ddl.py @@ -43,8 +43,8 @@ def main(): from rhosocial.activerecord.backend.impl.postgres.config import PostgresConnectionConfig config = PostgresConnectionConfig( - host="db-dev-1-n.rho.im", - port=15432, + host="localhost", + port=5432, database="test_db", username="root", password="password", diff --git a/docs/examples/chapter_08_scenarios/parallel_workers/README.md b/docs/examples/chapter_08_scenarios/parallel_workers/README.md index 418edac1..2307d330 100644 --- a/docs/examples/chapter_08_scenarios/parallel_workers/README.md +++ b/docs/examples/chapter_08_scenarios/parallel_workers/README.md @@ -96,7 +96,7 @@ python setup_db.py --async # 异步初始化（效果相同） ```text === 同步初始化模式 === 连接信息： - 主机：db-dev-1-n.rho.im:15432 + 主机：localhost:5432 数据库：test_db 用户：postgres diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/ddl/constraint.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/ddl/constraint.py index 43bb9ba8..f9886a1b 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/ddl/constraint.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/ddl/constraint.py @@ -81,14 +81,14 @@ class PostgresConstraintMixin(_ConstraintMixinBase): a fact apart from a guess. Measurements were taken against the three instances - ``tests/config/postgres_scenarios.yaml`` names, all on ``192.168.1.3``, + ``tests/config/postgres_scenarios.yaml`` names, each on its own host, with psycopg 3.3.6, each in a transaction that was rolled back: - * ``postgres_19beta4`` -- port 16690, ``server_version_num`` 190000, + * ``postgres_19beta4`` -- ``server_version_num`` 190000, ``PostgreSQL 19beta4 (Debian 19~beta4-1.pgdg13+1) on aarch64-unknown-linux-gnu`` - * ``postgres_18`` -- port 16689, ``server_version_num`` 180006 - * ``postgres_17`` -- port 16688, ``server_version_num`` 170011 + * ``postgres_18`` -- ``server_version_num`` 180006 + * ``postgres_17`` -- ``server_version_num`` 170011 Only ``TEMP`` tables inside rolled-back transactions were used. diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/named_connection/example_connections.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/named_connection/example_connections.py index ee3d0ed0..029ebd28 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/named_connection/example_connections.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/named_connection/example_connections.py @@ -11,8 +11,8 @@ def postgres_18(database: str = "test_db"): """PostgreSQL 18 development server connection.""" return PostgresConnectionConfig( - host="db-dev-1-n.rho.im", - port=15441, + host="localhost", + port=5432, database=database, username="root", password="password", @@ -25,8 +25,8 @@ def postgres_18_with_pool(pool_size: int = 5): if isinstance(pool_size, str): pool_size = int(pool_size) return PostgresConnectionConfig( - host="db-dev-1-n.rho.im", - port=15441, + host="localhost", + port=5432, database="test_db", username="root", password="password", @@ -38,8 +38,8 @@ def postgres_18_with_pool(pool_size: int = 5): def postgres_18_readonly(): """PostgreSQL 18 read-only connection (shorter timeout).""" return PostgresConnectionConfig( - host="db-dev-1-n.rho.im", - port=15441, + host="localhost", + port=5432, database="test_db", username="root", password="password", diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/named_connection/test_resolver.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/named_connection/test_resolver.py index 26c123a0..8b5f912d 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/named_connection/test_resolver.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/named_connection/test_resolver.py @@ -127,8 +127,8 @@ def test_postgres_18_connection(self): {}, ) assert isinstance(config, PostgresConnectionConfig) - assert config.host == "db-dev-1-n.rho.im" - assert config.port == 15441 + assert config.host == "localhost" + assert config.port == 5432 assert config.database == "test_db" assert config.sslmode == "prefer" diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_ddl_alter_config.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_ddl_alter_config.py index 6e820396..6a5bd10d 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_ddl_alter_config.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_ddl_alter_config.py @@ -332,7 +332,7 @@ class TestPostgresConstraintDeferral: ``[DEFERRABLE ...]`` as a tail property shared by every branch; the server contradicts that for CHECK and NOT NULL. Values below are the measured accept surface of PostgreSQL 19beta4 - (``192.168.1.3:16690/test_db``), per "backend facts win over + (the ``postgres_19beta4`` scenario server), per "backend facts win over documentation". SQLSTATEs are recorded on each switch's docstring. """ diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_postgres_constraint_capabilities.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_postgres_constraint_capabilities.py index 684683b5..623f812e 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_postgres_constraint_capabilities.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_postgres_constraint_capabilities.py @@ -2,14 +2,14 @@ """PostgreSQL constraint capability switches, pinned to live measurements. Every value asserted here was measured against the instances -``tests/config/postgres_scenarios.yaml`` names, all on ``192.168.1.3``, +``tests/config/postgres_scenarios.yaml`` names, each on its own host``, through psycopg 3.3.6, each probe inside a transaction that was rolled back: -* ``postgres_19beta4`` -- port 16690, ``server_version_num`` 190000, +* ``postgres_19beta4`` -- ``server_version_num`` 190000, ``PostgreSQL 19beta4 (Debian 19~beta4-1.pgdg13+1) on aarch64-unknown-linux-gnu`` -* ``postgres_18`` -- port 16689, ``server_version_num`` 180006 -* ``postgres_17`` -- port 16688, ``server_version_num`` 170011 +* ``postgres_18`` -- ``server_version_num`` 180006 +* ``postgres_17`` -- ``server_version_num`` 170011 The statements and the server's own SQLSTATEs are recorded in the docstrings of ``rhosocial.activerecord.backend.impl.postgres.mixins.ddl.constraint``; this From 7c47488805d75c587ab4318a994b97a961db8ca2 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sat, 7 Nov 2026 19:54:45 +0800 Subject: [PATCH 20/27] refactor: one definition for every value two places were writing The profile parser already said, in its own docstring, that source is ESTIMATE whenever the vintage is known and UNKNOWN when it is not -- and then wrote the words "estimate" and "unknown" into the field. It now assigns ProfileSource.ESTIMATE and ProfileSource.UNKNOWN, which is the same two values a consumer reads and a closed set a checker can name; the JSON artifact is unchanged because the enum is a str subclass, and the parser test asserts the members by identity as well as by value. This repository is one of the engines that override core's sample_column_values so it can read pg_stats instead of scanning the rows, and both overrides wrote limit: int = 100 -- a third and fourth spelling of the value core already owns as VALUE_SAMPLE_LIMIT. Both now reference that one name, so the twins still offer a default and offer the same object; a new test reads both signatures by inspect and asserts identity against core's constant, because a second 100 would compare equal and still be a second place to move. No CLI here ever defined DEFAULT_VALUE_SAMPLE_LIMIT and no CLI passed a limit, so there was nothing to delete on that half, and this repository prints no aggregate-profile note, so there was no sentence to inline. The output choices were spelled four times: introspect, query and status each carried the four-format list and info carried a two-format list under the same name. The four-format list is now OUTPUT_CHOICES in cli/output.py beside the provider factory that must be able to build every member, the narrower one is INFO_OUTPUT_CHOICES in the same file so the name says which surface it belongs to, and each subcommand imports the list it offers. Ruff, the CLI suites and the roundtrip matrix pass; the data-profile tests and anything that needs a server are for CI. --- .../backend/impl/postgres/cli/info.py | 6 +- .../backend/impl/postgres/cli/introspect.py | 4 +- .../backend/impl/postgres/cli/output.py | 17 +++++ .../backend/impl/postgres/cli/query.py | 4 +- .../backend/impl/postgres/cli/status.py | 4 +- .../postgres/introspection/introspector.py | 12 ++- .../test_introspection_profile_source.py | 75 +++++++++++++++++++ 7 files changed, 106 insertions(+), 16 deletions(-) create mode 100644 tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_profile_source.py diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/info.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/info.py index b820730f..cd5b6cd5 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/info.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/info.py @@ -73,12 +73,10 @@ ) from .connection import add_connection_args, add_version_arg, resolve_connection_config_from_args -from .output import create_provider, RICH_AVAILABLE +from .output import INFO_OUTPUT_CHOICES, create_provider, RICH_AVAILABLE logger = logging.getLogger(__name__) -OUTPUT_CHOICES = ['table', 'json'] - # Groups that are specific to PostgreSQL dialect DIALECT_SPECIFIC_GROUPS = {"PostgreSQL Native", "PostgreSQL Extensions"} @@ -164,7 +162,7 @@ def create_parser(subparsers): # Output format (only table and json; info has nested output) parser.add_argument( '-o', '--output', - choices=OUTPUT_CHOICES, + choices=INFO_OUTPUT_CHOICES, default='table', help='Output format (default: table)', ) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py index ba1a16de..d765fe95 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py @@ -27,9 +27,7 @@ from rhosocial.activerecord.backend.introspection.status import ExtensionInfo from .connection import add_connection_args, resolve_connection_config_from_args -from .output import create_provider - -OUTPUT_CHOICES = ["table", "json", "csv", "tsv"] +from .output import OUTPUT_CHOICES, create_provider #: CLI type -> the dialect capability switch that answers for it. #: diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/output.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/output.py index 6c2e1334..d15c0271 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/output.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/output.py @@ -14,6 +14,23 @@ RichOutputProvider = None # type: ignore[misc,assignment] +#: The output formats the introspect, query and status subcommands offer. +#: +#: The provider factory below is what makes them available: a format listed +#: here must map to a provider, and a provider nobody is offered is unreachable. +#: Defining the list once, beside the factory that has to be able to produce +#: every member, is what keeps "offered" and "producible" from drifting apart. +OUTPUT_CHOICES = ["table", "json", "csv", "tsv"] + +#: The narrower set ``info`` offers. +#: +#: ``info`` prints one object's facts, a shape the plain-text provider lays out +#: better than a record table; offering csv and tsv there would advertise two +#: renderings of a single row. The name says which surface it belongs to, so +#: the two lists are not one name answering two questions. +INFO_OUTPUT_CHOICES = ["table", "json"] + + def create_provider(output_format: str, ascii_borders: bool = False): """Factory function to get the correct output provider. diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py index ff637b79..881119db 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/query.py @@ -14,9 +14,7 @@ from rhosocial.activerecord.backend.errors import ConnectionError, QueryError from .connection import add_connection_args, resolve_connection_config_from_args -from .output import create_provider, RICH_AVAILABLE - -OUTPUT_CHOICES = ['table', 'json', 'csv', 'tsv'] +from .output import OUTPUT_CHOICES, create_provider, RICH_AVAILABLE def create_parser(subparsers): diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py index 17769175..daa06558 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/status.py @@ -18,9 +18,7 @@ from rhosocial.activerecord.backend.errors import ConnectionError, QueryError from .connection import add_connection_args, resolve_connection_config_from_args -from .output import create_provider, RICH_AVAILABLE - -OUTPUT_CHOICES = ['table', 'json', 'csv', 'tsv'] +from .output import OUTPUT_CHOICES, create_provider, RICH_AVAILABLE STATUS_TYPES = ["all", "config", "performance", "connections", "storage", "databases", "users"] diff --git a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py index 67e06226..c9cef3bb 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/introspection/introspector.py @@ -34,6 +34,7 @@ AsyncAbstractIntrospector, ) from rhosocial.activerecord.backend.introspection.cache import IntrospectionCache +from rhosocial.activerecord.backend.introspection.defaults import VALUE_SAMPLE_LIMIT from rhosocial.activerecord.backend.introspection.executor import ( IntrospectorExecutor, AsyncIntrospectorExecutor, @@ -53,6 +54,7 @@ TableType, IntrospectionTriggerInfo, IntrospectionViewInfo, + ProfileSource, ) from rhosocial.activerecord.backend.expression.types import DataType from .status_introspector import ( @@ -414,7 +416,11 @@ def parse_column_data_profiles( min_value=min_value, max_value=max_value, row_count=row_count, - source="estimate" if collected_at is not None else "unknown", + source=( + ProfileSource.ESTIMATE + if collected_at is not None + else ProfileSource.UNKNOWN + ), collected_at=str(collected_at) if collected_at is not None else None, correlation=( float(row["correlation"]) @@ -864,7 +870,7 @@ def sample_column_values( self, table_name: str, column_name: str, - limit: int = 100, + limit: int = VALUE_SAMPLE_LIMIT, schema: Optional[str] = None, ): """List a column's most frequent values, at most ``limit`` of them. @@ -1251,7 +1257,7 @@ async def sample_column_values( self, table_name: str, column_name: str, - limit: int = 100, + limit: int = VALUE_SAMPLE_LIMIT, schema: Optional[str] = None, ): """List a column's most frequent values, at most ``limit``; async twin. diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_profile_source.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_profile_source.py new file mode 100644 index 00000000..384b556c --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_profile_source.py @@ -0,0 +1,75 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_introspection_profile_source.py +"""The provenance of a profile, and the default both twins offer. + +``ProfileSource`` is core's closed set for where a profile number came from. +PostgreSQL is the engine that reads an estimate out of a catalogue, so it is +the one that has to name the members rather than spell their words: the same +``null_fraction`` means something different here than on an engine that counted +the rows, and the field is what carries that. + +The second half is the value-sample limit. This introspector overrides core's +``sample_column_values`` so it can read ``pg_stats`` instead of scanning the +rows, and an override that dropped the parameter's default would make +``introspector.sample_column_values(table, column)`` fail here while it +succeeds on every backend that inherits core's. Both twins therefore keep the +default, and both point at core's one definition of it; the assertion is by +identity, because a second ``100`` compares equal and is still a second place +to move. +""" + +import inspect + +from rhosocial.activerecord.backend.introspection.defaults import VALUE_SAMPLE_LIMIT +from rhosocial.activerecord.backend.introspection.types import ProfileSource +from rhosocial.activerecord.backend.impl.postgres.introspection.introspector import ( + AsyncPostgreSQLIntrospector, + PostgreSQLIntrospector, + PostgreSQLIntrospectorMixin, +) + + +class _MixinOnly(PostgreSQLIntrospectorMixin): + """The profile parser needs no backend; it reads the rows it is given. + + The parser is a method because it shares the static helpers below it in the + mixin, not because it reads a connection. + """ + + +class TestTheProfileSourceIsAMember: + def test_a_measured_vintage_is_an_estimate(self): + profile = _MixinOnly().parse_column_data_profiles( + [{"column_name": "status", "last_analyze": "2026-01-01 00:00:00"}], + "public", + "orders", + )[0] + + assert profile.source is ProfileSource.ESTIMATE + assert profile.source == "estimate" + + def test_an_unknown_vintage_is_unknown(self): + profile = _MixinOnly().parse_column_data_profiles( + [{"column_name": "status"}], + "public", + "orders", + )[0] + + assert profile.source is ProfileSource.UNKNOWN + assert profile.source == "unknown" + + +class TestTheValueSampleLimitHasOneOwner: + def test_both_twins_offer_core_default(self): + sync_default = ( + inspect.signature(PostgreSQLIntrospector.sample_column_values) + .parameters["limit"] + .default + ) + async_default = ( + inspect.signature(AsyncPostgreSQLIntrospector.sample_column_values) + .parameters["limit"] + .default + ) + + assert sync_default is VALUE_SAMPLE_LIMIT + assert async_default is VALUE_SAMPLE_LIMIT From 304c302f4590dfdbe619bd6e0b27b98ad9899e27 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 8 Nov 2026 01:43:53 +0800 Subject: [PATCH 21/27] fix(packaging): name the core line this code actually imports This backend's introspection modules import AbstractIntrospector, ProfileSource, ColumnDataProfile, introspection.defaults.VALUE_SAMPLE_LIMIT and the nine new introspection query statement classes, none of which exist in core 1.0.0.dev29. The >=1.0.0.dev0 floor admitted dev29, which is the version PyPI publishes and what pip install resolved, so the install succeeded and the first import failed. The pyproject floor is now >=1.0.0.dev30,<2.0.0, the line that actually carries these symbols, so a fresh resolve either gets a core that imports or stops instead of quietly installing a broken one. requirements-3.8.txt, the Python 3.8 install path, also names core with no floor at all, which resolved dev29 by the same accident; it now carries the same honest floor. The extras deliberately carry no core floor of their own: pip installs extras on top of the main dependencies, so they inherit this floor rather than resolving core independently. --- pyproject.toml | 2 +- requirements-3.8.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 02bc1f61..49b52aab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ # psycopg 3.3.6 requires Python 3.10+; 3.8 and 3.9 stay on the 3.2 line. "psycopg[binary]>=3.3.6; python_version >= '3.10'", "psycopg[binary]>=3.2.13; python_version < '3.10'", - "rhosocial-activerecord>=1.0.0.dev0,<2.0.0" + "rhosocial-activerecord>=1.0.0.dev30,<2.0.0" ] [project.optional-dependencies] diff --git a/requirements-3.8.txt b/requirements-3.8.txt index 8510f370..551a3d97 100644 --- a/requirements-3.8.txt +++ b/requirements-3.8.txt @@ -1,2 +1,2 @@ psycopg[binary]>=3.2.13 -rhosocial-activerecord +rhosocial-activerecord>=1.0.0.dev30,<2.0.0 From ad3d488f11178933c1510d9d0b73ac93284cf8c5 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 8 Nov 2026 02:16:53 +0800 Subject: [PATCH 22/27] refactor(cli): decide the NAME rule where a table can be asserted The NAME rule lived in twenty hand-written ifs, two per type per chain, each recomputed where it was needed and none of them asserted anywhere. Four of the listings over the statistics collector's scope -- unused-indexes, table-stats, runtime, index-stats -- had no check at all, so a name given beside one was parsed and dropped and the run printed the family listing under a one-object question. The module docstring already promised NAME was "never silently dropped"; the branches did not keep it. _NAME_RULE declares the answer once per type, and _validate_name reads it at one point in each chain, so a type added without a rule is a type nobody has decided for rather than one that happens to forget. The rule is now pinned three ways: as data (the whole table), as a set (every advertised type has an entry), and as behaviour (each category through a run). The four counter listings move from silently dropping a name to refusing it with exit 2 and the sentence the other family listings already used. Every type that was refused still is, with the same code and the same "does not take an object name" wording; every type that required a name still does, with the same code and the same "Table name is required" sentence. The refusal order is unchanged -- it stays after the capability gate and the stray-flag check, where the branches used to make it, so a run against an unreachable server still reports the connection rather than the argument. No changelog fragment: no user-visible behaviour changes except the four families that used to swallow a name. --- .../backend/impl/postgres/cli/introspect.py | 185 ++++++++---------- .../test_introspection_cli_names.py | 132 +++++++++++++ 2 files changed, 213 insertions(+), 104 deletions(-) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py index d765fe95..c1080da7 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/cli/introspect.py @@ -11,7 +11,9 @@ A type either lists a family or, when it names a single catalogue object, answers for that one object; ``NAME`` is required, optional or refused -according to which of the two it is, and never silently dropped. +according to which of the two it is, and never silently dropped. The rule per +type lives in ``_NAME_RULE`` and is applied by one validator, so a family +cannot be added without deciding what its NAME means. """ import argparse @@ -79,6 +81,53 @@ INTROSPECT_TYPES = list(_TYPE_CAPABILITIES) +#: What the optional NAME positional means for each type. +#: +#: ``required`` -- the type asks about one table and cannot be answered without +#: it. ``optional`` -- a catalogue family that can also answer for one member, +#: so a given name narrows it (and a name that matches nothing is an error, not +#: an empty list). ``forbidden`` -- the family has no single-member question +#: here, so a name would read as a filter that matched everything. +#: +#: The order is the order ``_TYPE_CAPABILITIES`` declares and the parser and +#: ``--help`` print, and the two tables cover the same keys: a type offered +#: without a rule is a type whose NAME decision nobody has made. +#: +#: ``tables`` stays one. Answering for one table would fill in its columns, +#: indexes and foreign keys, which is the aggregate this command deliberately +#: dropped. ``table-storage`` is the opposite case: its three figures are that +#: table's own, so the name *is* the question. +_NAME_RULE = { + "tables": "forbidden", + "views": "optional", + "columns": "required", + "indexes": "required", + "foreign-keys": "required", + "triggers": "optional", + "database": "forbidden", + "extensions": "optional", + "materialized-views": "optional", + "sequences": "optional", + "table-storage": "required", + # The four listings over the statistics collector's scope describe what the + # connection can see, not one object in it, so none takes a name -- the way + # ``tables`` does not. ``dependencies`` is not one of them: one object's + # edges are a real question, so a name narrows rather than being refused. + "unused-indexes": "forbidden", + "table-stats": "forbidden", + "runtime": "forbidden", + "index-stats": "forbidden", + "dependencies": "optional", + # The one-object families: the DDL that recreates one table, the profiles + # of its columns, and the samples of one of them. + "ddl": "required", + "column-profiles": "required", + "value-samples": "required", + # ``join-coverage`` asks about two tables; the left one is NAME, so it is + # required here the way the right one is required on its own flag. + "join-coverage": "required", +} + def _require_capability(backend, args) -> None: """Refuse a type the connected server cannot answer. @@ -105,16 +154,32 @@ def _require_capability(backend, args) -> None: sys.exit(2) -def _refuse_name(args: argparse.Namespace, hint: str) -> None: - """Refuse a ``NAME`` the type has no question for. +def _validate_name(args: argparse.Namespace) -> None: + """Apply the NAME rule table, once, instead of leaving it to each branch. - ``tables`` lists every table and ``database`` describes the database the - run connected to; neither can be narrowed to one object. Dropping the - name instead would print the family listing under a one-object question, - which is a different answer than the one asked for. + A silently ignored argument is the worst of the three outcomes: the run + succeeds, prints a full listing, and reads as though it had been narrowed. + So the table above is read here, once -- a branch that forgot would answer + a wider question than the one asked and look successful doing it. + + A *missing* name is an incomplete invocation and leaves with 1, the code a + missing ``--database`` gets; a name the type has no question for is a + request this command declines and leaves with 2. """ - print(f"Error: '{args.type}' does not take an object name; {hint}", file=sys.stderr) - sys.exit(2) + rule = _NAME_RULE.get(args.type) + if rule == "required" and not args.name: + print( + f"Error: Table name is required for '{args.type}' introspection", + file=sys.stderr, + ) + sys.exit(1) + if rule == "forbidden" and args.name: + print( + f"Error: '{args.type}' does not take an object name; " + "it answers for the whole database", + file=sys.stderr, + ) + sys.exit(2) def _refuse_unused_arguments(args: argparse.Namespace) -> None: @@ -278,7 +343,9 @@ def create_parser(subparsers): "and for value-samples and join-coverage (the left table); " "optional for triggers (a table), dependencies (one object) " "and for views, materialized-views, sequences and extensions " - "(one object); refused for tables and database", + "(one object); refused for tables, database and the four counter " + "listings (unused-indexes, table-stats, runtime, index-stats). " + "The rule per type is _NAME_RULE", ) # The three families that ask about a column or a second table carry the # names the probe needs. They stay flags rather than more positionals @@ -389,15 +456,11 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): backend.introspect_and_adapt() _require_capability(backend, args) _refuse_unused_arguments(args) + _validate_name(args) introspector = backend.introspector if args.type == "tables": - if args.name: - _refuse_name( - args, - "use 'columns', 'indexes' or 'foreign-keys' with a table name", - ) tables = introspector.list_tables(schema=None, include_system=args.include_system) data = _serialize_for_output(tables) provider.display_results(data, title="Tables") @@ -416,28 +479,16 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): provider.display_results(data, title="Views") elif args.type == "columns": - if not args.name: - print("Error: Table name is required for 'columns' introspection", file=sys.stderr) - sys.exit(1) columns = introspector.list_columns(args.name, schema=None) data = _serialize_for_output(columns) provider.display_results(data, title=f"Columns of {args.name}") elif args.type == "indexes": - if not args.name: - print("Error: Table name is required for 'indexes' introspection", file=sys.stderr) - sys.exit(1) indexes = introspector.list_indexes(args.name, schema=None) data = _serialize_for_output(indexes) provider.display_results(data, title=f"Indexes of {args.name}") elif args.type == "foreign-keys": - if not args.name: - print( - "Error: Table name is required for 'foreign-keys' introspection", - file=sys.stderr, - ) - sys.exit(1) fks = introspector.list_foreign_keys(args.name, schema=None) data = _serialize_for_output(fks) provider.display_results(data, title=f"Foreign Keys of {args.name}") @@ -448,8 +499,6 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): provider.display_results(data, title="Triggers") elif args.type == "database": - if args.name: - _refuse_name(args, "it describes the database the run connected to") info = introspector.get_database_info() data = _serialize_for_output(info) provider.display_results([data], title="Database Info") @@ -505,14 +554,8 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): # question asks the listing and picks the row -- one read, and # none of the three per-table questions get_table_info would add # (columns, indexes, foreign keys) that a storage answer does not - # need. The name is required because every figure is that - # table's own. - if not args.name: - print( - "Error: Table name is required for 'table-storage' introspection", - file=sys.stderr, - ) - sys.exit(1) + # need. ``_validate_name`` has already refused a run without one: + # every figure below is that table's own. tables = introspector.list_tables(schema=None, include_system=args.include_system) info = next((t for t in tables if t.name == args.name), None) if info is None: @@ -562,9 +605,6 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): # own flag. elif args.type == "ddl": - if not args.name: - print("Error: Table name is required for 'ddl' introspection", file=sys.stderr) - sys.exit(1) statement = introspector.get_ddl_statement(args.name, object_type=None, schema=None) if statement is None: # Not "no such table": the object exists, and PostgreSQL @@ -576,23 +616,11 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): _display_ddl(provider, statement, args) elif args.type == "column-profiles": - if not args.name: - print( - "Error: Table name is required for 'column-profiles' introspection", - file=sys.stderr, - ) - sys.exit(1) profiles = introspector.list_column_data_profiles(args.name, schema=None) data = _serialize_for_output(profiles) provider.display_results(data, title=f"Column profiles of {args.name}") elif args.type == "value-samples": - if not args.name: - print( - "Error: Table name is required for 'value-samples' introspection", - file=sys.stderr, - ) - sys.exit(1) if not args.column: print( "Error: --column is required for 'value-samples' introspection", @@ -604,12 +632,6 @@ def _handle_introspect_sync(args, backend: PostgresBackend, provider): provider.display_results(data, title=f"Values of {args.name}.{args.column}") elif args.type == "join-coverage": - if not args.name: - print( - "Error: Left table name is required for 'join-coverage' introspection", - file=sys.stderr, - ) - sys.exit(1) if not args.column: print( "Error: --column is required for 'join-coverage' introspection", @@ -662,15 +684,11 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider await backend.introspect_and_adapt() _require_capability(backend, args) _refuse_unused_arguments(args) + _validate_name(args) introspector = backend.introspector if args.type == "tables": - if args.name: - _refuse_name( - args, - "use 'columns', 'indexes' or 'foreign-keys' with a table name", - ) tables = await introspector.list_tables(schema=None, include_system=args.include_system) data = _serialize_for_output(tables) provider.display_results(data, title="Tables") @@ -689,28 +707,16 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider provider.display_results(data, title="Views") elif args.type == "columns": - if not args.name: - print("Error: Table name is required for 'columns' introspection", file=sys.stderr) - sys.exit(1) columns = await introspector.list_columns(args.name, schema=None) data = _serialize_for_output(columns) provider.display_results(data, title=f"Columns of {args.name}") elif args.type == "indexes": - if not args.name: - print("Error: Table name is required for 'indexes' introspection", file=sys.stderr) - sys.exit(1) indexes = await introspector.list_indexes(args.name, schema=None) data = _serialize_for_output(indexes) provider.display_results(data, title=f"Indexes of {args.name}") elif args.type == "foreign-keys": - if not args.name: - print( - "Error: Table name is required for 'foreign-keys' introspection", - file=sys.stderr, - ) - sys.exit(1) fks = await introspector.list_foreign_keys(args.name, schema=None) data = _serialize_for_output(fks) provider.display_results(data, title=f"Foreign Keys of {args.name}") @@ -721,8 +727,6 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider provider.display_results(data, title="Triggers") elif args.type == "database": - if args.name: - _refuse_name(args, "it describes the database the run connected to") info = await introspector.get_database_info() data = _serialize_for_output(info) provider.display_results([data], title="Database Info") @@ -779,12 +783,6 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider elif args.type == "table-storage": # The async twin of the sync branch: the same listing read, # awaited, and the same pick. - if not args.name: - print( - "Error: Table name is required for 'table-storage' introspection", - file=sys.stderr, - ) - sys.exit(1) tables = await introspector.list_tables(schema=None, include_system=args.include_system) info = next((t for t in tables if t.name == args.name), None) if info is None: @@ -828,9 +826,6 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider provider.display_results(data, title=title) elif args.type == "ddl": - if not args.name: - print("Error: Table name is required for 'ddl' introspection", file=sys.stderr) - sys.exit(1) statement = await introspector.get_ddl_statement( args.name, object_type=None, schema=None ) @@ -840,23 +835,11 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider _display_ddl(provider, statement, args) elif args.type == "column-profiles": - if not args.name: - print( - "Error: Table name is required for 'column-profiles' introspection", - file=sys.stderr, - ) - sys.exit(1) profiles = await introspector.list_column_data_profiles(args.name, schema=None) data = _serialize_for_output(profiles) provider.display_results(data, title=f"Column profiles of {args.name}") elif args.type == "value-samples": - if not args.name: - print( - "Error: Table name is required for 'value-samples' introspection", - file=sys.stderr, - ) - sys.exit(1) if not args.column: print( "Error: --column is required for 'value-samples' introspection", @@ -868,12 +851,6 @@ async def _handle_introspect_async(args, backend: AsyncPostgresBackend, provider provider.display_results(data, title=f"Values of {args.name}.{args.column}") elif args.type == "join-coverage": - if not args.name: - print( - "Error: Left table name is required for 'join-coverage' introspection", - file=sys.stderr, - ) - sys.exit(1) if not args.column: print( "Error: --column is required for 'join-coverage' introspection", diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_names.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_names.py index a629528e..a08e990e 100644 --- a/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_names.py +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/introspection/test_introspection_cli_names.py @@ -6,6 +6,13 @@ and the ones that list a family refuse it. Nothing here needs a server: the fake backend says what the catalogue would answer, so these tests pin the dispatch rather than PostgreSQL's own bookkeeping. + +The rule itself is a table, ``_NAME_RULE``, and the tests below pin it two +ways: once as data, so a change to any one type's answer is a visible edit +rather than a drift, and once as behaviour, so the table is what the run +actually applies. The set-level assertion is the third: a type advertised +without a rule is a type whose NAME decision nobody has made, which is the +failure the table exists to prevent. """ import argparse @@ -236,6 +243,10 @@ class TestNameIsRefused: [ (["introspect", "tables", "users"], "'tables'"), (["introspect", "database", "mydb"], "'database'"), + (["introspect", "unused-indexes", "users"], "'unused-indexes'"), + (["introspect", "table-stats", "users"], "'table-stats'"), + (["introspect", "runtime", "users"], "'runtime'"), + (["introspect", "index-stats", "users"], "'index-stats'"), ], ) def test_a_family_listing_refuses_a_name(self, run_cli, argv, marker): @@ -247,3 +258,124 @@ def test_a_family_listing_refuses_a_name(self, run_cli, argv, marker): def test_a_refused_name_never_reaches_the_catalogue(self, run_cli): run_cli(["introspect", "tables", "users", "--database", "mydb"]) assert FakeBackend.last_instance.introspector.calls == [] + + +class TestTheNameRuleTable: + """The table, pinned: as data, and as the rule the run applies. + + A table nobody asserts is text nobody guarantees. The first test is the + guard the cross-repository clause of ``.claude/rules/constants.md`` asks + for -- a private name is legitimate because the module reads it in + several places, and the test is what makes the reading checkable. The + second is the set-level half: a new type cannot be added without + deciding what its NAME means. + """ + + def test_every_advertised_type_has_a_rule(self): + assert set(introspect_mod._NAME_RULE) == set(introspect_mod.INTROSPECT_TYPES) + + def test_the_table_says_what_each_types_name_means(self): + assert introspect_mod._NAME_RULE == { + "tables": "forbidden", + "views": "optional", + "columns": "required", + "indexes": "required", + "foreign-keys": "required", + "triggers": "optional", + "database": "forbidden", + "extensions": "optional", + "materialized-views": "optional", + "sequences": "optional", + "table-storage": "required", + "unused-indexes": "forbidden", + "table-stats": "forbidden", + "runtime": "forbidden", + "index-stats": "forbidden", + "dependencies": "optional", + "ddl": "required", + "column-profiles": "required", + "value-samples": "required", + "join-coverage": "required", + } + + def test_the_table_covers_no_type_the_parser_does_not_offer(self): + assert set(introspect_mod._NAME_RULE) - set(introspect_mod.INTROSPECT_TYPES) == set() + + @pytest.mark.parametrize( + "type_", + [ + "tables", + "database", + "unused-indexes", + "table-stats", + "runtime", + "index-stats", + ], + ) + def test_a_refused_type_is_refused_by_the_table(self, type_, capsys): + with pytest.raises(SystemExit) as excinfo: + introspect_mod._validate_name(argparse.Namespace(type=type_, name="users")) + + assert excinfo.value.code == 2 + assert f"'{type_}' does not take an object name" in capsys.readouterr().err + + @pytest.mark.parametrize( + "type_", + [ + "columns", + "indexes", + "foreign-keys", + "table-storage", + "ddl", + "column-profiles", + "value-samples", + "join-coverage", + ], + ) + def test_a_required_name_is_required_by_the_table(self, type_, capsys): + """A missing required NAME is an incomplete invocation, not a refusal. + + It exits 1 like a missing ``--database``; 2 is reserved for a request + this command declines to serve. + """ + with pytest.raises(SystemExit) as excinfo: + introspect_mod._validate_name(argparse.Namespace(type=type_, name=None)) + + assert excinfo.value.code == 1 + assert f"Table name is required for '{type_}'" in capsys.readouterr().err + + @pytest.mark.parametrize( + "type_", + [ + "views", + "triggers", + "extensions", + "materialized-views", + "sequences", + "dependencies", + ], + ) + def test_an_optional_name_may_be_given_or_not(self, type_): + introspect_mod._validate_name(argparse.Namespace(type=type_, name=None)) + introspect_mod._validate_name(argparse.Namespace(type=type_, name="orders")) + + +class TestANameIsRequiredEndToEnd: + """The required rule, applied by a run rather than read off the table.""" + + @pytest.mark.parametrize( + "type_", + [ + "columns", + "indexes", + "foreign-keys", + "table-storage", + "ddl", + "column-profiles", + ], + ) + def test_a_run_without_the_name_exits_one_and_asks_nothing(self, run_cli, type_, capsys): + _, err, exc = run_cli(["introspect", type_, "--database", "mydb"]) + assert exc is not None and exc.code == 1 + assert f"Table name is required for '{type_}'" in err + assert FakeBackend.last_instance.introspector.calls == [] From ea90ce71124c50b3be7d32edc397c75011766a34 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 8 Nov 2026 10:20:19 +0800 Subject: [PATCH 23/27] ci: install the core branch before the requirements that would resolve it The floor is right; the order that enforced it was not. Each matrix job installed the requirements and extras first, and the Python 3.8 jobs install requirements-3.8.txt, which names the core floor directly. pip therefore resolved rhosocial-activerecord from PyPI, whose newest release is 1.0.0.dev29, and stopped with an unsatisfiable >=1.0.0.dev30,<2.0.0 before the step that installs the matching core from git ever ran. The other jobs had the same order without anything in their files naming the core, so the guarantee was an accident of which file each interpreter happened to read. The order was already wrong before the floor moved: the old >=1.0.0.dev0 floor let the wrong core install silently and moved the failure to import time. Every job in this workflow now installs the core branch and the testsuite pin from git before the requirements and extras. Those files are unchanged, constraints included; pip now finds an installed 1.0.0.dev30 that satisfies >=1.0.0.dev30,<2.0.0 and installs only the pieces still missing from the index, instead of looking for a core version PyPI does not have. The floor, the version, the git refs and the testsuite pin are untouched; the install order is the only change. --- .github/workflows/test.yml | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6fbc38b..c93f77e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,16 +61,16 @@ jobs: with: python-version: "3.15" - - name: Install dependencies - run: | - pip install -r requirements.txt - pip install -r requirements-dev.txt - - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install -r requirements-dev.txt + - name: Create postgres scenario config run: | cat > postgres_scenarios.yaml << EOF @@ -276,16 +276,16 @@ jobs: python-version: ${{ matrix.python-version }} freethreaded: ${{ matrix.freethreaded }} - - name: Install dependencies - run: | - pip install -r requirements.txt - pip install -r requirements-dev.txt - - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install -r requirements-dev.txt + - name: Create postgres scenario config run: | cat > postgres_scenarios.yaml << EOF @@ -383,6 +383,11 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install activerecord and testsuite + run: | + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" + pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies run: | if [ "${{ matrix.python-version }}" == "3.8" ]; then @@ -393,11 +398,6 @@ jobs: pip install -r requirements-dev.txt fi - - name: Install activerecord and testsuite - run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" - pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - - name: Create postgres scenario config run: | cat > postgres_scenarios.yaml << EOF @@ -486,16 +486,16 @@ jobs: with: python-version: "3.14" - - name: Install dependencies - run: | - pip install -r requirements.txt - pip install -r requirements-dev.txt - - name: Install activerecord and testsuite run: | pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install -r requirements-dev.txt + - name: Create postgres scenario config run: | cat > postgres_scenarios.yaml << EOF From 8bc40223ccf54d208d7b75589e93e573a42e19a3 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 8 Nov 2026 12:00:58 +0800 Subject: [PATCH 24/27] docs: state what PostgreSQL answers, including what pg_stats cannot give Core defines the probes and the layers; this backend's page now states what PostgreSQL itself answers. It adds the twenty-one-probe matrix, the six operational families read from the statistics collector, and the three data families read from pg_stats, including the four encodings that surface as traps: anyarray bounds through array_to_json, signed n_distinct, -1 as unknown, and a histogram that omits the kept most-common values. The operational section names each source and its tense, since the counters are only since the last statistics reset, and states that DDL is an assembly with supports_ddl_extraction_native false. The data section states when the profile reports ESTIMATE or UNKNOWN and why an asynchronous last_analyze usually leaves the timestamp unset, and why a never-analysed table answers with no rows. Join coverage is described as a two-sided measurement over distinct non-null values, with NULLIF so an empty column answers None and zero overlap answers 0.0, and as a double scan that is never implicit. The CLI documentation gains the nine family types plus table-storage, the NAME rule for each, and the note that ddl legitimately reports no available text and exits non-zero. The Chinese mirror follows the same headings and carries the same code blocks, now in English. --- .../introspection.md | 461 +++++++++++++++++- .../introspection.md | 416 ++++++++++++---- 2 files changed, 780 insertions(+), 97 deletions(-) diff --git a/docs/en_US/postgres_specific_features/introspection.md b/docs/en_US/postgres_specific_features/introspection.md index e27c1143..349c35bf 100644 --- a/docs/en_US/postgres_specific_features/introspection.md +++ b/docs/en_US/postgres_specific_features/introspection.md @@ -13,13 +13,81 @@ The PostgreSQL introspection system is accessible via `backend.introspector` and - **Foreign Key Information**: Referenced table, column mapping, update/delete actions - **View Information**: View definition SQL - **Trigger Information**: Trigger event, execution timing +- **Operational Measurements**: Unused indexes, table and index usage counters, live runtime metrics, dependencies, and DDL +- **Column Data Profiles**: Per-column statistics from `pg_stats`, kept value samples, and join coverage + +## What PostgreSQL Answers + +Core declares twenty-one `supports_*` probes, and each asks a question about the +engine rather than about the library. PostgreSQL answers nineteen of them `True`. +The table below is this backend's answer to every one, in the order core declares +them, with the catalogue view or statement each answer is read from. + +| Probe | Answer | PostgreSQL's answer comes from | +|-------|--------|-------------------------------| +| `supports_introspection` | `True` | This dialect's own declaration; all seven introspection scopes are declared | +| `supports_database_info` | `True` | `pg_database`, `version()`, `pg_database_size()` | +| `supports_table_introspection` | `True` | `pg_class` / `pg_namespace`, with `pg_stat_user_tables` left-joined for live rows | +| `supports_column_introspection` | `True` | `pg_attribute`, `pg_attrdef`, `pg_index`, `information_schema.columns` | +| `supports_index_introspection` | `True` | `pg_index`, `pg_class`, `pg_am` | +| `supports_foreign_key_introspection` | `True` | `pg_constraint` (`contype = 'f'`) | +| `supports_view_introspection` | `True` | `pg_class` (`relkind = 'v'`), `pg_get_viewdef()` | +| `supports_trigger_introspection` | `True` | `pg_trigger`, `pg_class`, `pg_proc` | +| `supports_runtime_stats` | `True` | `pg_stat_activity` | +| `supports_index_stats` | `True` | `pg_stat_user_indexes`, `pg_relation_size()` | +| `supports_table_stats` | `True` | `pg_stat_user_tables` | +| `supports_unused_indexes_detection` | `True` | `pg_stat_user_indexes.idx_scan` | +| `supports_partition_info` | `False` | Inherited core default; no partition-information formatter (see below) | +| `supports_object_dependencies` | `True` | `pg_depend`, `pg_class`, `pg_namespace` | +| `supports_column_data_profile` | `True` | `pg_stats` | +| `supports_column_value_sample` | `True` | `pg_stats.most_common_vals` | +| `supports_join_coverage` | `True` | Counts distinct values on both sides; no catalogue source | +| `supports_ddl_extraction` | `True` | `pg_class` / `pg_namespace`; the projection carries no DDL text | +| `supports_extensions` | `True` | `pg_extension`, through `supports_extension_introspection` | +| `supports_table_storage_introspection` | `True` | `pg_total_relation_size()`, `pg_relation_size()`, `pg_indexes_size()` | +| `supports_ddl_extraction_native` | `False` | No single native DDL retrieval call (see below) | + +Two answers are `False`. One reason is written in the probe itself; the other +is inherited from core's default, and what the default leaves out is stated below. + +### The Two False Answers + +`supports_partition_info` answers `False` through core's default: this dialect +does not override the probe and declares no partition-information query +expression or formatter. A partitioned table is still answered — the table +listing reports it with the `table_type` label `PARTITIONED TABLE`, from the +`p` relkind code — but that is the relation's kind, not the partition tree: +nothing here reports the parent, the children or the bounds. Partition DDL is a +separate protocol family (`supports_hash_partitioning`, `supports_attach_partition` +and their neighbours), not this introspection probe. + +`supports_ddl_extraction_native` answers `False`. The probe's docstring is the +reason: PostgreSQL has no single native DDL retrieval call. `pg_get_expr` and +`pg_get_viewdef` recover expressions, but a faithful `CREATE TABLE` also needs +storage parameters, the tablespace and per-column clauses the catalogue keeps +only in parts, so what this backend can offer is an assembly, and `native=False` +is how an assembly is marked. `supports_ddl_extraction` stays `True` because the +family is declared and a statement renders for it; the statement itself is +described under `get_ddl_statement` below. + +### The Probes PostgreSQL Adds + +Three more probes belong to this backend rather than to core's protocol, and +each answers `True`: `supports_extension_introspection` reads `pg_extension`, +`supports_materialized_view_introspection` reads `pg_matviews`, and +`supports_sequence_introspection` reads `information_schema.sequences` with +`pg_sequences` left-joined where the server has it. `supports_extensions` is +core's spelling of the first one and delegates to it rather than declaring a +second `True`, so the two names are one answer: what PostgreSQL has installed, +not which extensions the dialect knows about. The extension listing's rows come +from `pg_extension` through the introspector. ## Basic Usage ### Accessing the Introspector ```python -rhosocial.activerecord.backend.impl.postgres.backend import PostgresBackend +from rhosocial.activerecord.backend.impl.postgres.backend import PostgresBackend backend = PostgresBackend( host="localhost", port=5432, @@ -33,6 +101,10 @@ backend.connect() introspector = backend.introspector ``` +`backend.introspector` is a `PostgreSQLIntrospector`; the async backend's is an +`AsyncPostgreSQLIntrospector`. Both are built on `PostgreSQLIntrospectorMixin`, +so every method below exists on both under the same name. + ### Getting Database Information ```python @@ -216,6 +288,11 @@ for idx in indexes: print(f"GiST index: {idx.name} (likely for geospatial data)") ``` +`index_type` starts as the catalogue's `pg_am.amname`. Core's `IndexType` enum +has no `SPGIST` or `BRIN` member, so the parser maps the catalogue's `spgist` +to `IndexType.GIST` and `brin` to `IndexType.BTREE`; the value in the listing is +a core enum member, not the `amname` spelling. + ### Trigger Events PostgreSQL triggers support multiple events: @@ -248,12 +325,272 @@ for idx in indexes: print(f"{idx.name}.{col.name}: {nulls_order}") ``` +## Operational Introspection + +The declaration layer answers what the server *is*. The six families here answer +what it has *been doing*, and they read the statistics collector's views rather +than `pg_catalog` proper. The tense is PostgreSQL's: every counter is since the +statistics were last reset — a crash or a `pg_stat_reset()` — so a zero means +nothing happened in the window, never nothing ever. A number without that +sentence is not a fact, which is why the tense is stated here for every family. + +| Method | Reads | Returns | +|--------|-------|---------| +| `list_unused_indexes(schema=None)` | `pg_stat_user_indexes` where `idx_scan = 0` | Schema, table and index address | +| `list_table_usage_stats(schema=None)` | `pg_stat_user_tables` | Sequential and index scans, insert/update/delete counts, live-row estimate | +| `list_runtime_metrics(schema=None)` | `pg_stat_activity` | One row per backend process | +| `list_index_usage_stats(schema=None)` | `pg_stat_user_indexes`, `pg_relation_size()` | Scans, rows read, rows fetched, index bytes | +| `list_object_dependencies(object_name=None, schema=None)` | `pg_depend`, `pg_class`, `pg_namespace` | Dependency edges | +| `get_ddl_statement(object_name, object_type=None, schema=None)` | `pg_class`, `pg_namespace`; `pg_get_expr` / `pg_get_viewdef` recover expressions | Object address and kind; no text today, and an assembly is marked `native=False` | + +### `list_unused_indexes` + +PostgreSQL ships no vendor view for unused indexes, so the listing is computed +from the counters: `pg_stat_user_indexes` rows with `idx_scan = 0` in the +requested schema, joined to `pg_namespace` for the schema name. The projection +is the address — `schema_name`, `table_name`, `index_name` — because the address +is what the family's common shape carries, and the counter that selected the row +is zero by definition. + +A zero counter means no index scan since the statistics were last reset. That is +PostgreSQL's tense for this family and a weaker claim than the word "unused" +suggests: an index untouched since a crash, or since a `pg_stat_reset()`, reports +zero. The answer is "worth reviewing", never "safe to drop". The listing only +covers indexes the statistics view has a row for; an index it does not list is +not thereby proven used. + +### `list_table_usage_stats` + +The listing reads `pg_stat_user_tables` for the requested schema, one row per +table the collector has seen. It projects `seq_scan`, `idx_scan`, `n_tup_ins`, +`n_tup_upd`, `n_tup_del` and `n_live_tup`. The view's other facts — dead tuples, +the last-access times — are real but are not projected: they have no common slot +in `TableUsageStats`, and where they go is the backend reader's decision rather +than the common shape's. A table with no statistics row is absent from the +listing, which is the honest scope rather than a zero-filled row. A zero in +`seq_scan` and a zero in `n_tup_ins` support different conclusions, and neither +survives being read as "no activity at all". + +### `list_runtime_metrics` + +The listing reads `pg_stat_activity` and answers one row per backend process. +It takes no table name: a runtime measurement is about the server, and a table +filter would discard rows that were never about a table. The projection is +`application_name` as `name`, `state` as `value`, `wait_event_type` as `kind`, +`wait_event` as `description`, and `backend_start` as `source`. `source` dates +the session the row describes rather than labelling a counter window; the counter +window is the statistics window stated above, the same for every row. +`pg_stat_statements` is deliberately not read: it is an extension, and answering +from a catalogue that is only sometimes installed would make the family depend +on a module rather than on the engine. + +### `list_index_usage_stats` + +The same `pg_stat_user_indexes` view, read for how much rather than whether. The +projection is `idx_scan` (scans), `idx_tup_read` (rows read), `idx_tup_fetch` +(rows fetched) and `pg_relation_size(indexrelid)` (index bytes) in one row. The +size call is why "how much use" and "how much space" land together instead of +two queries a caller has to join. Every counter is since the statistics were last +reset; a zero here is a measurement, not an absence. The listing covers the +indexes the statistics view has rows for. + +### `list_object_dependencies` + +The graph is read from `pg_depend`, joined to `pg_class` on `refobjid` and to +`pg_namespace` for the schema. Each row describes an edge in which a relation is +the referenced object; `object_name`, when given, narrows the listing to that +relation's edges — the "what breaks if I drop this" question. The projection is +`schema_name`, the referenced relation's `relname` as `referenced`, `deptype` as +`kind`, and `classid` as `referenced_type`. `action` stays unset: PostgreSQL has +no per-edge action column, and a dependant's behaviour on drop is a property of +its class, so the formatter does not guess one from `deptype`. There is no +`deptype` filter either; the kind is reported so the reader can decide which +edges to act on. + +### `get_ddl_statement` + +`supports_ddl_extraction` answers `True` and a statement renders, but the +statement this backend renders is an address, not text. It selects +`pg_class.relname`, `pg_namespace.nspname` and `pg_class.relkind` for the named +relation, and projects no `ddl` column. The parser skips a row carrying no text, +so `get_ddl_statement` returns `None` for a table that exists: "cannot reproduce +this", not "no such table". `supports_ddl_extraction_native` answers `False`, so +a caller who needs the server's own text knows in advance that any assembly here +would be a reconstruction; `pg_get_expr` and `pg_get_viewdef` can recover +expressions, but a faithful `CREATE TABLE` would also need storage parameters, +tablespace and per-column clauses the catalogue keeps only in parts. The CLI +reports `Error: no DDL available for ''` and exits `1`; it does not print +an empty row, because a row with no text would read as an answer. + +## Column Data Profiles + +The three data families read `pg_stats`, the per-column statistics `ANALYZE` +leaves, not the rows themselves. `pg_stats` holds one row per analysed column, +so the profile costs one catalogue query however large the table is — a catalogue +read rather than a pass over the rows, after the cheap column listing core reads +first. The price is that every figure is an estimate of the last `ANALYZE`, which +is why the answer carries its source and the time it was collected. + +| Method | Reads | Returns | +|--------|-------|---------| +| `list_column_data_profiles(table_name, schema=None)` | One `pg_stats` row per analysed column | Null fraction, distinct count, extremes, row estimate, correlation, and the kept value list | +| `sample_column_values(table_name, column_name, limit=100, schema=None)` | `pg_stats.most_common_vals` and `most_common_freqs` | Value and frequency pairs | +| `estimate_join_coverage(left_table, left_column, right_table, right_column, schema=None)` | The rows of both tables | Two coverage fractions | + +### What `pg_stats` Will Not Give + +`pg_stats` declares `histogram_bounds` and `most_common_vals` as `anyarray`, the +pseudo-type that stands for "an array of whatever this column's type is", and +keeps `most_common_freqs` beside them. PostgreSQL refuses to subscript an +`anyarray` column (`cannot subscript type anyarray`) and defines no `MIN`/`MAX` +aggregate over one, because a type without a btree opclass has no aggregate. +`array_to_json` is declared over `anyarray`, so it is the one call that accepts +the columns as they are; this backend reads all three arrays through it. The +cost is that a bound arrives as a JSON scalar: a number stays a number, while a +date, a timestamp or a UUID reads as its ISO text. A typed endpoint is not +obtainable from `pg_stats` without knowing the column's type at plan time, which +is a limit of the catalogue rather than a choice of the library. Everything else +on the profile — `null_frac`, `n_distinct`, `correlation`, `reltuples` — is +native. + +### The Encodings the Parser Converts + +`n_distinct` is signed. A negative value is not a count but the fraction of the +rows that are distinct — ANALYZE writes it that way when the column's cardinality +grows with the table — so the parser multiplies it by `reltuples` to produce the +count. A non-negative value is already a count and is carried through. Without +the conversion a unique column on a million-row table would report `-1` rather +than a million. + +`-1` is "unknown", not minus one of anything. `null_frac` is `-1` on a column +ANALYZE has not measured since the statistics were reset, and `reltuples` is +`-1` for a relation the planner has never scanned; the parser turns both into +`None`. A caller that reads a negative fraction as a fraction, or a negative +row count as a count, would divide by a number that means "nobody looked". + +`histogram_bounds` excludes the values ANALYZE kept as most-common. Its first +and last elements are therefore the extremes of what is left, not of the column: +a status column whose minimum is also its most common value has that value in +`most_common_vals`, not at the histogram's edge. The parser merges the kept +`most_common_vals` back in before calling the endpoints `min_value` and +`max_value`. Only like kinds are reconciled — two numbers, or two strings, the +latter covering text and the ISO spellings of dates and UUIDs — because a JSON +string and a JSON number have no meaningful order between them. + +`pg_stats` has a row only for a column ANALYZE has seen. A column with no +statistics row is simply not in the profile list, so the answer is one entry per +analysed column, not one entry per declared column; read it beside +`list_columns`, not in place of it. + +### The Source This Backend Reports + +Every profile this backend returns carries `ProfileSource.ESTIMATE` when +`pg_stat_all_tables.last_analyze` or `last_autoanalyze` is recorded, and +`ProfileSource.UNKNOWN` when neither is. Both figures are estimates either way; +the rule is that an estimate carries the time it was collected, and `UNKNOWN` is +how a profile says its vintage is not known. + +The timestamp is often absent for a reason worth knowing: PostgreSQL's +statistics collector writes `last_analyze` asynchronously, so a profile read +immediately after `ANALYZE` usually finds no timestamp yet. A `pg_stat_reset()` +also erases the time while leaving the statistics in place. Neither is an error, +and neither licenses inventing a date. The join from `pg_stats` to +`pg_stat_all_tables` is a LEFT join for this reason: a table the collector has +not seen still yields its `pg_stats` rows when it has any, with an unknown +vintage. + +A table that was never analysed has no `pg_stats` row, so it answers with no +rows at all. That is "not measured" as an absence, not a profile with zeros in +its slots. + +```python +from rhosocial.activerecord.backend.introspection.types import ProfileSource + +profiles = backend.introspector.list_column_data_profiles("orders") +for profile in profiles: + print( + profile.column_name, + profile.null_fraction, + profile.distinct_count, + profile.min_value, + profile.max_value, + profile.row_count, + profile.source, + profile.collected_at, + ) + if profile.source is ProfileSource.ESTIMATE: + print(" measured at", profile.collected_at) +``` + +### Sampling a Column's Values + +`sample_column_values` projects the two arrays ANALYZE keeps in lockstep: +`most_common_vals` and `most_common_freqs`. The parser pairs them by position, so +the i-th frequency belongs to the i-th value, and a missing frequency stays +`None`. The arrays are kept side by side rather than unnested in SQL because +their element types differ; keeping them apart preserves the values in the +column's own types. ANALYZE keeps at most `statistics_target` values (100 by +default), so a low-cardinality column's listing is usually complete while a +high-cardinality one is a sample of the head; fewer values than requested is the +normal answer, not a truncated one. This backend slices to the requested `limit` +after reading the catalogue, since unnesting two arrays to pair and limit them +in SQL would cost more than the slice. `frequency` is populated; `count` stays +`None`, because `pg_stats` reports shares rather than row counts. NULL is a +legitimate member of the domain and is kept. + +### Measuring Join Coverage + +PostgreSQL answers `supports_join_coverage`, and the statement it renders is the +only family here that names two user tables rather than one catalogue relation; +both identifiers are quoted by the dialect. The row carries two fractions, +`left_coverage` and `right_coverage`, each computed as the share of one column's +distinct non-null values that also occurs in the other column's distinct non-null +values. Distinct on both sides is what makes the figure a coverage of values +rather than of rows: a key repeated a thousand times that matches once would +otherwise read as well covered. + +NULLs are excluded from the numerators and the denominators alike. In the +numerator the exclusion is load-bearing, because `SELECT DISTINCT` treats a NULL +as one value; in the divisor `COUNT(DISTINCT ...)` skips NULLs on its own, but +the statement says so explicitly because the share is of the values that could +join. `NULLIF(COUNT(DISTINCT ...), 0)` then makes a column with no value at all +answer NULL rather than `0.0`: "nothing matched" and "nothing to match" stay +apart. Zero overlap is a measurement and comes back as `0.0`. The `* 1.0` in +each fraction is also load-bearing: PostgreSQL divides integers by truncation, +so `COUNT(*) / COUNT(...)` would answer zero for every share below one. + +Both fractions are reported because the relation is not symmetric. A left +coverage of `1.0` beside a right coverage of `0.2` says every tested value +exists on the other side while most of that other domain is unaccounted for; +one similarity score would average the two and describe neither. Collations are +not reconciled: values the database would treat as equal under one collation may +be counted as distinct here, so the fractions are a lower bound across +mismatched collations, never an upper one. + +The cost is a double scan plus an intersection of the two distinct sets, which +is why the probe answers `True` on the capability and the call is never implicit. +Profiling a table does not measure any join coverage, listing tables does not, +and no schema pass asks for it on the way past. `estimate_join_coverage` returns +`None` when the row carries no fraction at all; the CLI reports that and exits +`1` rather than printing a row of nulls that would read as a measured zero. + +```python +coverage = backend.introspector.estimate_join_coverage( + "orders", "buyer_id", + "users", "id", +) +if coverage is None: + print("no measurement") +else: + print(coverage.left_coverage, coverage.right_coverage) +``` + ## Async API The async backend provides identical introspection methods with the same names as the sync version: ```python -rhosocial.activerecord.backend.impl.postgres.backend.async_backend import AsyncPostgresBackend +from rhosocial.activerecord.backend.impl.postgres.backend.async_backend import AsyncPostgresBackend backend = AsyncPostgresBackend( host="localhost", port=5432, @@ -389,6 +726,33 @@ async def get_schema_info(backend): | `clear_cache()` | Clear cache | None | | `invalidate_cache(scope, ...)` | Invalidate cache | `scope`, `name`, `table_name` | +### PostgreSQL-Specific Methods + +| Method | Description | Parameters | +|--------|-------------|------------| +| `list_materialized_views()` | List materialized views | `schema`, `include_system` | +| `get_materialized_view_info(name)` | Get one materialized view | `name`, `schema` | +| `materialized_view_exists(name)` | Check a materialized view exists | `name`, `schema` | +| `list_sequences()` | List sequences | `schema` | +| `get_sequence_info(name)` | Get one sequence | `name`, `schema` | +| `sequence_exists(name)` | Check a sequence exists | `name`, `schema` | +| `list_extensions()` | List installed extensions | None | +| `get_extension_info(name)` | Get one installed extension | `name` | +| `extension_exists(name)` | Check an extension is installed | `name` | +| `list_unused_indexes()` | Indexes with no scan since the statistics reset | `schema` | +| `list_table_usage_stats()` | Per-table usage counters | `schema` | +| `list_runtime_metrics()` | One row per backend process | `schema` | +| `list_index_usage_stats()` | Per-index counters and size | `schema` | +| `list_object_dependencies()` | Dependency edges, or one object's | `object_name`, `schema` | +| `get_ddl_statement(name)` | The DDL text, or `None` on this backend | `name`, `object_type`, `schema` | +| `list_column_data_profiles(table_name)` | Per-column profile from `pg_stats` | `table_name`, `schema` | +| `sample_column_values(table_name, column_name)` | Kept values and frequencies | `table_name`, `column_name`, `limit`, `schema` | +| `estimate_join_coverage(...)` | Two coverage fractions | `left_table`, `left_column`, `right_table`, `right_column`, `schema` | + +The three data methods are also declared on core's introspector and use the same +names; the PostgreSQL reader supplies the parsing, and the async twin awaits the +same names. + ## Command Line Introspection The PostgreSQL backend provides command-line introspection commands to query database metadata without writing code. @@ -459,11 +823,54 @@ python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ --database mydb --user postgres --schema auth ``` +### Querying Operational and Data Families + +The six operational families and the three data families are their own types: + +```bash +# Indexes no query has used since the statistics were reset +python -m rhosocial.activerecord.backend.impl.postgres introspect unused-indexes \ + --database mydb --user postgres + +# Per-table usage counters and per-index counters +python -m rhosocial.activerecord.backend.impl.postgres introspect table-stats \ + --database mydb --user postgres +python -m rhosocial.activerecord.backend.impl.postgres introspect index-stats \ + --database mydb --user postgres + +# What the server has been doing +python -m rhosocial.activerecord.backend.impl.postgres introspect runtime \ + --database mydb --user postgres + +# The dependency graph, or one object's edges +python -m rhosocial.activerecord.backend.impl.postgres introspect dependencies orders \ + --database mydb --user postgres + +# The DDL that recreates a table (this backend answers "no DDL available") +python -m rhosocial.activerecord.backend.impl.postgres introspect ddl orders \ + --database mydb --user postgres + +# Profiles and samples from pg_stats +python -m rhosocial.activerecord.backend.impl.postgres introspect column-profiles orders \ + --database mydb --user postgres +python -m rhosocial.activerecord.backend.impl.postgres introspect value-samples orders \ + --column status --database mydb --user postgres + +# Overlap between two columns' value domains +python -m rhosocial.activerecord.backend.impl.postgres introspect join-coverage orders \ + --column buyer_id --right-table users --right-column id \ + --database mydb --user postgres + +# One table's storage split +python -m rhosocial.activerecord.backend.impl.postgres introspect table-storage orders \ + --database mydb --user postgres +``` + ### Introspection Types | Type | Description | NAME | |------|-------------|------| -| `tables` | List every base table | Refused | +| `tables` | Every relation in the schema: base tables, views, materialized views, foreign tables and partitioned tables | Refused | | `views` | List ordinary views, or one by name | Optional: a view name | | `columns` | One table's columns | Required: a table name | | `indexes` | One table's indexes | Required: a table name | @@ -473,14 +880,50 @@ python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ | `extensions` | Installed extensions, or one by name | Optional: an extension name | | `materialized-views` | Materialized views, or one by name | Optional: a view name | | `sequences` | Sequences, or one by name | Optional: a sequence name | +| `table-storage` | One table's storage split and row estimate | Required: a table name | +| `unused-indexes` | Indexes with `idx_scan = 0` since the statistics were reset | Refused | +| `table-stats` | Per-table usage counters | Refused | +| `runtime` | One row per backend process from `pg_stat_activity` | Refused | +| `index-stats` | Per-index counters and size | Refused | +| `dependencies` | Dependency edges, or one object's | Optional: an object name | +| `ddl` | The DDL that recreates one table | Required: a table name | +| `column-profiles` | Per-column profiles from `pg_stats` | Required: a table name | +| `value-samples` | A column's kept values and frequencies | Required: a table name; `--column` required | +| `join-coverage` | Two columns' value-domain overlap | Required: the left table; `--column`, `--right-table`, `--right-column` required | + +Ten rows here are new relative to the ten types this page listed before; nine of +them are the families above. `table-storage` asks the declaration layer for one +table's storage split; `unused-indexes`, `table-stats`, `runtime`, `index-stats`, +`dependencies` and `ddl` are the six operational families; `column-profiles`, +`value-samples` and `join-coverage` are the three data families. Each is offered +because the probe switch named in `_TYPE_CAPABILITIES` answers `True` on the +adapted server; a type whose switch is `False` exits `2` with "not supported by +this server" rather than returning an empty list. `NAME` selects the object the type answers for, and each type has exactly one -answer to whether it takes one. `columns`, `indexes` and `foreign-keys` name the -table they belong to and require it; `triggers` accepts a table name and lists -every trigger when it gets none; `views`, `materialized-views`, `sequences` and -`extensions` accept an object name and list the family when they get none. -`tables` and `database` take no name: a name there is an error rather than a -filter, so a question aimed at one object cannot quietly return the whole family. +answer to whether it takes one. `columns`, `indexes`, `foreign-keys`, +`table-storage`, `ddl`, `column-profiles`, `value-samples` and `join-coverage` +name the table they are about and require it — for `join-coverage` that is the +left table. `triggers` accepts a table name, `dependencies` accepts an object +name, and `views`, `materialized-views`, `sequences` and `extensions` accept an +object name; each lists its whole family when it gets none. `tables`, `database`, +`unused-indexes`, `table-stats`, `runtime` and `index-stats` take no name: a name +there is an error rather than a filter, because the counter listings answer for +the connection's scope rather than for one object. + +Three types ask about a column or a second table on their own flags. +`value-samples` takes `--column`, and `join-coverage` takes `--column`, +`--right-table` and `--right-column`; a missing one is an incomplete invocation +and exits `1`. A flag given to a type that has no question for it exits `2` +rather than being ignored, the same rule NAME follows. + +`ddl` is the one type that can legitimately report a failure on an existing +table: this backend's projection carries no DDL text, so `get_ddl_statement` +returns `None` and the command prints `Error: no DDL available for ''` +and exits `1`. It does not print an empty row, because a row with no text would +read as an answer. `supports_ddl_extraction_native` is `False` here, so a caller +that needs the server's own text already knows the answer is an assembly, not +a retrieval. A run exits `1` when it could not finish -- the connection failed, a query failed, or the object a name asked for is not there -- and `2` when the request diff --git a/docs/zh_CN/postgres_specific_features/introspection.md b/docs/zh_CN/postgres_specific_features/introspection.md index 8c78b380..83b06ed8 100644 --- a/docs/zh_CN/postgres_specific_features/introspection.md +++ b/docs/zh_CN/postgres_specific_features/introspection.md @@ -13,13 +13,55 @@ PostgreSQL 内省系统位于 `backend.introspector` 属性中，提供： - **外键信息**：引用表、列映射、更新/删除行为 - **视图信息**：视图定义 SQL - **触发器信息**：触发事件、执行时机 +- **运维测量**：未使用索引、表与索引的使用计数、实时运行指标、依赖关系与 DDL +- **列数据剖析**：来自 `pg_stats` 的逐列统计、保留值样本与连接覆盖率 + +## PostgreSQL 回答什么 + +核心声明了二十一个 `supports_*` 探针，每个探针问的是关于引擎的问题，而不是关于库的功能开关。PostgreSQL 其中十九个回答 `True`。下表是本后端对每一个探针的回答，顺序与核心声明的一致，并列出每个答案所读取的目录视图或语句。 + +| 探针 | 回答 | PostgreSQL 的答案来自 | +|------|------|----------------------| +| `supports_introspection` | `True` | 本方言自身的声明；七个内省作用域全部声明 | +| `supports_database_info` | `True` | `pg_database`、`version()`、`pg_database_size()` | +| `supports_table_introspection` | `True` | `pg_class` / `pg_namespace`，左连接 `pg_stat_user_tables` 取得活跃行数 | +| `supports_column_introspection` | `True` | `pg_attribute`、`pg_attrdef`、`pg_index`、`information_schema.columns` | +| `supports_index_introspection` | `True` | `pg_index`、`pg_class`、`pg_am` | +| `supports_foreign_key_introspection` | `True` | `pg_constraint`（`contype = 'f'`） | +| `supports_view_introspection` | `True` | `pg_class`（`relkind = 'v'`）、`pg_get_viewdef()` | +| `supports_trigger_introspection` | `True` | `pg_trigger`、`pg_class`、`pg_proc` | +| `supports_runtime_stats` | `True` | `pg_stat_activity` | +| `supports_index_stats` | `True` | `pg_stat_user_indexes`、`pg_relation_size()` | +| `supports_table_stats` | `True` | `pg_stat_user_tables` | +| `supports_unused_indexes_detection` | `True` | `pg_stat_user_indexes.idx_scan` | +| `supports_partition_info` | `False` | 继承自核心的默认值；没有分区信息格式化方法（见下） | +| `supports_object_dependencies` | `True` | `pg_depend`、`pg_class`、`pg_namespace` | +| `supports_column_data_profile` | `True` | `pg_stats` | +| `supports_column_value_sample` | `True` | `pg_stats.most_common_vals` | +| `supports_join_coverage` | `True` | 统计两侧的不同值；没有目录来源 | +| `supports_ddl_extraction` | `True` | `pg_class` / `pg_namespace`；投影不携带 DDL 文本 | +| `supports_extensions` | `True` | `pg_extension`，经由 `supports_extension_introspection` | +| `supports_table_storage_introspection` | `True` | `pg_total_relation_size()`、`pg_relation_size()`、`pg_indexes_size()` | +| `supports_ddl_extraction_native` | `False` | 没有单一的原生 DDL 获取调用（见下） | + +有两个回答是 `False`。一个原因写在探针自己身上；另一个继承自核心的默认值，默认值没有写出的部分在下文说明。 + +### 两个 False 回答 + +`supports_partition_info` 通过核心的默认值回答 `False`：本方言不覆盖该探针，也没有声明任何分区信息查询表达式或格式化方法。分区表仍然会被回答——表列表会用 `table_type` 标签 `PARTITIONED TABLE` 报告它，来自 `p` 这个 relkind 代码——但那是关系的种类，不是分区树：这里没有任何东西报告父表、子表或边界。分区 DDL 是另一个协议族（`supports_hash_partitioning`、`supports_attach_partition` 及其邻居），不是这个内省探针。 + +`supports_ddl_extraction_native` 回答 `False`。探针自己的文档字符串就是原因：PostgreSQL 没有单一的、原生的 DDL 获取调用。`pg_get_expr` 与 `pg_get_viewdef` 能恢复表达式，但一个忠实的 `CREATE TABLE` 还需要存储参数、表空间以及目录只分开保存的逐列子句，所以本后端能提供的只是一次组装，而 `native=False` 正是组装结果的标记方式。`supports_ddl_extraction` 仍为 `True`，因为该族已被声明且有为它渲染的语句；语句本身在下文 `get_ddl_statement` 处描述。 + +### PostgreSQL 额外的探针 + +还有三个探针属于本后端而非核心协议，且都回答 `True`：`supports_extension_introspection` 读取 `pg_extension`，`supports_materialized_view_introspection` 读取 `pg_matviews`，`supports_sequence_introspection` 读取 `information_schema.sequences`，并在服务器支持时左连接 `pg_sequences`。`supports_extensions` 是核心对第一个探针的拼写，它委托给后者而不是再声明一个 `True`，所以这两个名字是同一个答案：PostgreSQL 安装了什么，而不是方言知道哪些扩展。扩展列表的行通过内省器来自 `pg_extension`。 ## 基本用法 ### 访问内省器 ```python -rhosocial.activerecord.backend.impl.postgres.backend import PostgresBackend +from rhosocial.activerecord.backend.impl.postgres.backend import PostgresBackend backend = PostgresBackend( host="localhost", port=5432, @@ -29,82 +71,84 @@ backend = PostgresBackend( ) backend.connect() -# 通过 introspector 属性访问 +# Access via introspector attribute introspector = backend.introspector ``` +`backend.introspector` 是 `PostgreSQLIntrospector`；异步后端的则是 `AsyncPostgreSQLIntrospector`。两者都建立在 `PostgreSQLIntrospectorMixin` 之上，所以下文每个方法在两者上都以相同名字存在。 + ### 获取数据库信息 ```python -# 获取数据库基本信息 +# Get basic database information db_info = backend.introspector.get_database_info() -print(f"数据库名称: {db_info.name}") -print(f"版本: {db_info.version}") -print(f"编码: {db_info.encoding}") -print(f"数据库大小: {db_info.size_bytes} bytes") +print(f"Database name: {db_info.name}") +print(f"Version: {db_info.version}") +print(f"Encoding: {db_info.encoding}") +print(f"Database size: {db_info.size_bytes} bytes") ``` ### 列出表 ```python -# 列出所有用户表 +# List all user tables tables = backend.introspector.list_tables() for table in tables: - print(f"表: {table.name}, 类型: {table.table_type.value}") + print(f"Table: {table.name}, Type: {table.table_type.value}") if table.comment: - print(f" 注释: {table.comment}") + print(f" Comment: {table.comment}") -# 包含系统表 +# Include system tables all_tables = backend.introspector.list_tables(include_system=True) -# 过滤特定类型 +# Filter by specific type base_tables = backend.introspector.list_tables(table_type="BASE TABLE") views = backend.introspector.list_tables(table_type="VIEW") materialized_views = backend.introspector.list_tables(table_type="MATERIALIZED VIEW") -# 检查表是否存在 +# Check if table exists if backend.introspector.table_exists("users"): - print("users 表存在") + print("users table exists") -# 获取表详情 +# Get table details table_info = backend.introspector.get_table_info("users") if table_info: - print(f"表名: {table_info.name}") - print(f"估计行数: {table_info.row_count}") - print(f"表大小: {table_info.size_bytes} bytes") + print(f"Table name: {table_info.name}") + print(f"Estimated rows: {table_info.row_count}") + print(f"Table size: {table_info.size_bytes} bytes") ``` ### 查询列信息 ```python -# 列出表的所有列 +# List all columns of a table columns = backend.introspector.list_columns("users") for col in columns: nullable = "NOT NULL" if col.nullable.value == "NOT_NULL" else "NULLABLE" pk = " [PK]" if col.is_primary_key else "" print(f"{col.name}: {col.data_type} {nullable}{pk}") if col.comment: - print(f" 注释: {col.comment}") + print(f" Comment: {col.comment}") -# 获取主键信息 +# Get primary key information pk = backend.introspector.get_primary_key("users") if pk: - print(f"主键: {[c.name for c in pk.columns]}") + print(f"Primary key: {[c.name for c in pk.columns]}") -# 获取单列信息 +# Get single column information col_info = backend.introspector.get_column_info("users", "email") ``` ### 查询索引 ```python -# 列出表的所有索引 +# List all indexes of a table indexes = backend.introspector.list_indexes("users") for idx in indexes: idx_type = idx.index_type.value if idx.index_type else "BTREE" unique = "UNIQUE " if idx.is_unique else "" primary = "PRIMARY " if idx.is_primary else "" - print(f"{primary}{unique}索引: {idx.name} ({idx_type})") + print(f"{primary}{unique}Index: {idx.name} ({idx_type})") for col in idx.columns: desc = "DESC" if col.is_descending else "ASC" nulls = "" @@ -116,11 +160,11 @@ for idx in indexes: ### 查询外键 ```python -# 列出表的外键 +# List foreign keys of a table foreign_keys = backend.introspector.list_foreign_keys("posts") for fk in foreign_keys: - print(f"外键: {fk.name}") - print(f" 列: {fk.columns} -> {fk.referenced_table}.{fk.referenced_columns}") + print(f"FK: {fk.name}") + print(f" Columns: {fk.columns} -> {fk.referenced_table}.{fk.referenced_columns}") print(f" ON DELETE: {fk.on_delete.value}") print(f" ON UPDATE: {fk.on_update.value}") ``` @@ -128,29 +172,29 @@ for fk in foreign_keys: ### 查询视图 ```python -# 列出所有视图 +# List all views views = backend.introspector.list_views() for view in views: - print(f"视图: {view.name}") + print(f"View: {view.name}") -# 获取视图详情 +# Get view details view_info = backend.introspector.get_view_info("user_posts_summary") if view_info: - print(f"定义: {view_info.definition}") - print(f"可更新: {view_info.is_updatable}") + print(f"Definition: {view_info.definition}") + print(f"Updatable: {view_info.is_updatable}") ``` ### 查询触发器 ```python -# 列出所有触发器 +# List all triggers triggers = backend.introspector.list_triggers() for trigger in triggers: - print(f"触发器: {trigger.name} on {trigger.table_name}") - print(f" 事件: {trigger.event.value}") - print(f" 时机: {trigger.timing.value}") + print(f"Trigger: {trigger.name} on {trigger.table_name}") + print(f" Event: {trigger.event.value}") + print(f" Timing: {trigger.timing.value}") -# 列出特定表的触发器 +# List triggers for a specific table table_triggers = backend.introspector.list_triggers("users") ``` @@ -172,9 +216,9 @@ PostgreSQL 支持多种表类型： tables = backend.introspector.list_tables() for table in tables: if table.table_type.value == "MATERIALIZED VIEW": - print(f"物化视图: {table.name}") + print(f"Materialized view: {table.name}") elif table.table_type.value == "PARTITIONED TABLE": - print(f"分区表: {table.name}") + print(f"Partitioned table: {table.name}") ``` ### Schema 支持 @@ -182,13 +226,13 @@ for table in tables: PostgreSQL 使用 schema 组织数据库对象： ```python -# 默认 schema 为 'public' -tables = backend.introspector.list_tables() # 查询 public schema +# Default schema is 'public' +tables = backend.introspector.list_tables() # Queries public schema -# 指定其他 schema +# Specify a different schema tables = backend.introspector.list_tables(schema="analytics") -# 获取表详情时指定 schema +# Get table details with specific schema table_info = backend.introspector.get_table_info("users", schema="auth") ``` @@ -211,11 +255,13 @@ from rhosocial.activerecord.backend.introspection.types import IndexType indexes = backend.introspector.list_indexes("articles") for idx in indexes: if idx.index_type == IndexType.GIN: - print(f"GIN 索引: {idx.name} (可能用于 JSONB 或全文搜索)") + print(f"GIN index: {idx.name} (likely for JSONB or full-text search)") elif idx.index_type == IndexType.GIST: - print(f"GiST 索引: {idx.name} (可能用于地理数据)") + print(f"GiST index: {idx.name} (likely for geospatial data)") ``` +`index_type` 起初是目录中的 `pg_am.amname`。核心的 `IndexType` 枚举没有 `SPGIST` 或 `BRIN` 成员，所以解析器把目录中的 `spgist` 映射为 `IndexType.GIST`，把 `brin` 映射为 `IndexType.BTREE`；列表中的值是核心的枚举成员，而不是 `amname` 的拼写。 + ### 触发器事件 PostgreSQL 触发器支持多种事件： @@ -230,9 +276,9 @@ PostgreSQL 触发器支持多种事件： ```python triggers = backend.introspector.list_triggers("users") for trigger in triggers: - print(f"触发器 {trigger.name}:") - print(f" 事件: {trigger.event.value}") - print(f" 时机: {trigger.timing.value}") # BEFORE, AFTER, INSTEAD OF + print(f"Trigger {trigger.name}:") + print(f" Event: {trigger.event.value}") + print(f" Timing: {trigger.timing.value}") # BEFORE, AFTER, INSTEAD OF ``` ### NULLS 排序 @@ -248,12 +294,127 @@ for idx in indexes: print(f"{idx.name}.{col.name}: {nulls_order}") ``` +## 运维内省 + +声明层回答服务器*是什么*。这里的六个族回答它*一直在做什么*，它们读取统计收集器的视图，而不是 `pg_catalog` 本身。时态是 PostgreSQL 的：每个计数都是自统计上次被重置以来——一次崩溃或一次 `pg_stat_reset()`——所以零表示窗口内什么都没发生，永远不是“从来没有”。一个没有这句话的数字不是事实，因此这里为每个族都写明时态。 + +| 方法 | 读取 | 返回 | +|------|------|------| +| `list_unused_indexes(schema=None)` | `pg_stat_user_indexes` 中 `idx_scan = 0` 的行 | schema、表与索引地址 | +| `list_table_usage_stats(schema=None)` | `pg_stat_user_tables` | 顺序扫描与索引扫描、插入/更新/删除计数、活跃行估计 | +| `list_runtime_metrics(schema=None)` | `pg_stat_activity` | 每个后端进程一行 | +| `list_index_usage_stats(schema=None)` | `pg_stat_user_indexes`、`pg_relation_size()` | 扫描次数、读取行数、取回行数、索引字节数 | +| `list_object_dependencies(object_name=None, schema=None)` | `pg_depend`、`pg_class`、`pg_namespace` | 依赖边 | +| `get_ddl_statement(object_name, object_type=None, schema=None)` | `pg_class`、`pg_namespace`；`pg_get_expr` / `pg_get_viewdef` 恢复表达式 | 对象地址与种类；今天没有文本，若组装则标记 `native=False` | + +### `list_unused_indexes` + +PostgreSQL 没有为未使用索引提供厂商视图，所以该列表由计数计算而来：在所请求的 schema 中取 `pg_stat_user_indexes` 里 `idx_scan = 0` 的行，并连接 `pg_namespace` 取得 schema 名。投影是地址——`schema_name`、`table_name`、`index_name`——因为地址正是该族共同形状所携带的内容，而选出该行的计数按定义就是零。 + +零计数表示自统计上次重置以来没有发生索引扫描。这是 PostgreSQL 对这个族的时态，也比“未使用”这个词所暗示的断言更弱：一个自崩溃以来、或自 `pg_stat_reset()` 以来未被碰过的索引会报告零。答案只是“值得复查”，永远不是“可以安全删除”。该列表只覆盖统计视图中有行的索引；它没有列出的索引并不因此就被证明使用过。 + +### `list_table_usage_stats` + +该列表读取所请求 schema 的 `pg_stat_user_tables`，收集器见过的每张表一行。它投影 `seq_scan`、`idx_scan`、`n_tup_ins`、`n_tup_upd`、`n_tup_del` 与 `n_live_tup`。视图的其他事实——死元组、最后访问时间——是真实的但没有被投影：它们在 `TableUsageStats` 中没有共同的槽，它们该去哪里是后端读取器的决定，而不是共同形状的决定。没有统计行的表不会出现在列表中，这是诚实的范围，而不是补零的行。`seq_scan` 的零与 `n_tup_ins` 的零支持不同的结论，两者都不能被读成“完全没有活动”。 + +### `list_runtime_metrics` + +该列表读取 `pg_stat_activity`，每个后端进程回答一行。它不接受表名：运行时测量是关于服务器的，表过滤会丢弃那些从来与表无关的行。投影是 `application_name` 作为 `name`、`state` 作为 `value`、`wait_event_type` 作为 `kind`、`wait_event` 作为 `description`、`backend_start` 作为 `source`。`source` 标注的是该行所描述会话的起始时间，而不是计数窗口；计数窗口是上文所述的统计窗口，对每一行都相同。这里刻意不读取 `pg_stat_statements`：它是一个扩展，从一个只是有时安装的目录作答会让该族依赖模块而不是依赖引擎。 + +### `list_index_usage_stats` + +同一个 `pg_stat_user_indexes` 视图，这次读取的是用了多少而不是有没有用过。投影是在一行里的 `idx_scan`（扫描次数）、`idx_tup_read`（读取行数）、`idx_tup_fetch`（取回行数）与 `pg_relation_size(indexrelid)`（索引字节数）。正是这个大小调用让“用了多少”和“占了多少空间”落在同一行，而不是两条需要调用方自己连接的查询。每个计数都是自统计上次重置以来；这里的零是一个测量值，不是缺失。该列表覆盖统计视图中有行的索引。 + +### `list_object_dependencies` + +依赖图从 `pg_depend` 读入，按 `refobjid` 连接 `pg_class`，并按 schema 连接 `pg_namespace`。每一行描述一条某关系作为被引用对象的边；给出 `object_name` 时，列表收窄到该关系的边——也就是“删掉它会坏什么”的问题。投影是 `schema_name`、被引用关系的 `relname` 作为 `referenced`、`deptype` 作为 `kind`、`classid` 作为 `referenced_type`。`action` 保持为空：PostgreSQL 没有逐边的 action 列，依赖对象在删除时的行为是它所属类的属性，所以格式化方法不会从 `deptype` 猜测一个。这里也没有 `deptype` 过滤；kind 被报告出来，让读者自己决定要处理哪些边。 + +### `get_ddl_statement` + +`supports_ddl_extraction` 回答 `True`，也确实为它渲染了语句，但本后端渲染的语句是地址，不是文本。它为指定关系选择 `pg_class.relname`、`pg_namespace.nspname` 与 `pg_class.relkind`，并且不投影 `ddl` 列。解析器会跳过没有文本的行，所以 `get_ddl_statement` 对一张存在的表返回 `None`：这是“无法重现这个”，不是“没有这张表”。`supports_ddl_extraction_native` 回答 `False`，所以需要服务器原生文本的调用方事先就知道这里任何组装都只是重建；`pg_get_expr` 与 `pg_get_viewdef` 能恢复表达式，但一个忠实的 `CREATE TABLE` 还需要存储参数、表空间以及目录只分开保存的逐列子句。CLI 会报告 `Error: no DDL available for ''` 并以 `1` 退出；它不会打印空行，因为一行没有文本的记录会被读成一个答案。 + +## 列数据剖析 + +三个数据族读取 `pg_stats`，也就是 `ANALYZE` 留下的逐列统计，而不是行本身。`pg_stats` 每个已分析列一行，所以无论表多大，剖析本身都只花一条目录查询——这是一次目录读取，而不是对行的遍历，核心在此之前还会先读一次廉价的列清单。代价是每个数字都是上一次 `ANALYZE` 的估计，这也是答案要携带来源与收集时间的原因。 + +| 方法 | 读取 | 返回 | +|------|------|------| +| `list_column_data_profiles(table_name, schema=None)` | 每个已分析列一行 `pg_stats` | 空值比例、不同值计数、极值、行估计、相关性以及保留值列表 | +| `sample_column_values(table_name, column_name, limit=100, schema=None)` | `pg_stats.most_common_vals` 与 `most_common_freqs` | 值与频率对 | +| `estimate_join_coverage(left_table, left_column, right_table, right_column, schema=None)` | 两张表的行 | 两个覆盖率分数 | + +### `pg_stats` 不会给出什么 + +`pg_stats` 把 `histogram_bounds` 与 `most_common_vals` 声明为 `anyarray`——“这个列的类型所对应的数组”的伪类型，并在旁边保留 `most_common_freqs`。PostgreSQL 拒绝对 `anyarray` 列取下标（`cannot subscript type anyarray`），也没有为它定义 `MIN`/`MAX` 聚合，因为没有 btree 操作类的类型没有聚合。`array_to_json` 声明在 `anyarray` 上，所以它是唯一能原样接受这些列的调用；本后端通过它读取全部三个数组。代价是边界以 JSON 标量到达：数字仍是数字，而日期、时间戳或 UUID 会读成它的 ISO 文本。不知道计划时的列类型，就无法从 `pg_stats` 得到带类型的端点，这是目录的限制，而不是库的选择。剖析中的其他内容——`null_frac`、`n_distinct`、`correlation`、`reltuples`——都是原生类型。 + +### 解析器转换的编码 + +`n_distinct` 带符号。负值不是计数，而是不同值所占行的比例——当列基数随表增长时 ANALYZE 就这样写——所以解析器用 `reltuples` 乘以它来得到计数。非负值已经是一个计数，会原样携带。没有这个转换，一张百万行表上的唯一列会报告 `-1` 而不是一百万。 + +`-1` 是“未知”，不是任何东西的负一。`null_frac` 在统计被重置后 ANALYZE 尚未测量的列上是 `-1`，`reltuples` 在规划器从未扫描过的关系上是 `-1`；解析器把两者都变成 `None`。把负比例读成比例、把负行数读成计数的人，会除以一个表示“没人看过”的数字。 + +`histogram_bounds` 不包含 ANALYZE 作为最常见值保留的那些值。因此它的首尾元素是剩余部分的极值，而不是列的极值：一个最小值同时也是最常见值的状态列，该值在 `most_common_vals` 里，而不在直方图的边缘。解析器在把端点称为 `min_value` 与 `max_value` 之前，会把保留的 `most_common_vals` 合并回去。只有同类才会被比较——两个数字，或两个字符串，后者涵盖文本以及日期和 UUID 的 ISO 拼写——因为 JSON 字符串与 JSON 数字之间没有有意义的顺序。 + +`pg_stats` 只为 ANALYZE 见过的列保留行。没有统计行的列根本不在剖析列表中，所以答案是每个已分析列一条，而不是每个声明列一条；要把它与 `list_columns` 并排阅读，而不是取代后者。 + +### 本后端报告的来源 + +本后端返回的每个剖析，在记录了 `pg_stat_all_tables.last_analyze` 或 `last_autoanalyze` 时携带 `ProfileSource.ESTIMATE`，两者都没有时携带 `ProfileSource.UNKNOWN`。两种情况下数字都是估计；规则是估计必须携带它被收集的时间，而 `UNKNOWN` 是剖析表示其年代未知的方式。 + +这个时间戳经常缺失，原因值得知道：PostgreSQL 的统计收集器是异步写入 `last_analyze` 的，所以在 `ANALYZE` 之后立刻读取剖析，通常还找不到时间戳。`pg_stat_reset()` 也会在保留统计的同时抹掉时间。两者都不是错误，也都不允许凭空编一个日期。从 `pg_stats` 到 `pg_stat_all_tables` 的连接是 LEFT 连接，正是这个原因：收集器没见过的表，只要它有 `pg_stats` 行，仍会带着未知年代返回。 + +从未分析过的表没有 `pg_stats` 行，所以它完全不返回行。这是作为缺失的“未测量”，而不是槽位填零的剖析。 + +```python +from rhosocial.activerecord.backend.introspection.types import ProfileSource + +profiles = backend.introspector.list_column_data_profiles("orders") +for profile in profiles: + print( + profile.column_name, + profile.null_fraction, + profile.distinct_count, + profile.min_value, + profile.max_value, + profile.row_count, + profile.source, + profile.collected_at, + ) + if profile.source is ProfileSource.ESTIMATE: + print(" measured at", profile.collected_at) +``` + +### 采样一列的值 + +`sample_column_values` 投影 ANALYZE 成对保留的两个数组：`most_common_vals` 与 `most_common_freqs`。解析器按位置配对，所以第 i 个频率属于第 i 个值，缺失的频率保持 `None`。两个数组并排保留而不是在 SQL 中展开，因为它们的元素类型不同；分开保留能让值保持列自身的类型。ANALYZE 最多保留 `statistics_target` 个值（默认 100），所以低基数列的列表通常是完整的，而高基数列只是头部的一个样本；返回值少于请求的 `limit` 是正常答案，不是被截断。本后端在读取目录之后按请求的 `limit` 切片，因为在 SQL 中展开两个数组来配对并限制它们会比切片更贵。`frequency` 会被填充；`count` 保持 `None`，因为 `pg_stats` 报告的是比例而不是行数。NULL 是值域的合法成员，会被保留。 + +### 测量连接覆盖率 + +PostgreSQL 回答 `supports_join_coverage`，它渲染的语句是这里唯一命名两张用户表而不是一个目录关系的族；两个标识符都由方言加引号。行里携带两个分数，`left_coverage` 与 `right_coverage`，每个都是一列的不同非空值中也出现在另一列不同非空值中的比例。两侧都取不同值，才让这个数字成为值的覆盖而不是行的覆盖：一个重复一千次、只匹配一次的键，否则会被读成覆盖得很好。 + +NULL 从分子和分母中都被排除。在分子中这个排除是承重的，因为 `SELECT DISTINCT` 会把 NULL 当成一个值；在分母中 `COUNT(DISTINCT ...)` 自己会跳过 NULL，但语句仍显式写出，因为这个比例是“本可以连接的值”的比例。随后 `NULLIF(COUNT(DISTINCT ...), 0)` 让完全没有值的列回答 NULL 而不是 `0.0`：“什么都没匹配”与“没有东西可匹配”保持分开。零重叠是一个测量值，会以 `0.0` 返回。每个分数中的 `* 1.0` 同样是承重的：PostgreSQL 的整数除法会截断，所以 `COUNT(*) / COUNT(...)` 对任何小于一的份额都会回答零。 + +两个分数都报告，因为关系并不对称。左侧覆盖 `1.0` 而右侧覆盖 `0.2`，表示每个被测值都在另一边存在，而那边的大部分值域没有被涵盖；单一相似度分数会把两者平均，从而两者都不描述。排序规则不会被调和：数据库在某个排序规则下会视为相等的值，在这里可能被计为不同，所以这些分数在排序规则不匹配时是下界，永远不是上界。 + +代价是一次双重扫描加上两个不同值集合的交集，这也是探针在能力上回答 `True`、而调用从不隐式发生的原因。剖析一张表不会测量任何连接覆盖率，列表表也不会，任何 schema 遍历都不会顺路问它。当行里完全没有分数时，`estimate_join_coverage` 返回 `None`；CLI 会报告这一点并以 `1` 退出，而不是打印一行会被读成“测得零”的 null。 + +```python +coverage = backend.introspector.estimate_join_coverage( + "orders", "buyer_id", + "users", "id", +) +if coverage is None: + print("no measurement") +else: + print(coverage.left_coverage, coverage.right_coverage) +``` + ## 异步 API 异步后端提供相同的内省方法，方法名与同步版本相同： ```python -rhosocial.activerecord.backend.impl.postgres.backend.async_backend import AsyncPostgresBackend +from rhosocial.activerecord.backend.impl.postgres.backend.async_backend import AsyncPostgresBackend backend = AsyncPostgresBackend( host="localhost", port=5432, @@ -263,7 +424,7 @@ backend = AsyncPostgresBackend( ) await backend.connect() -# 异步内省方法（方法名与同步版本相同） +# Async introspection methods (same names as sync version) db_info = await backend.introspector.get_database_info() tables = await backend.introspector.list_tables() columns = await backend.introspector.list_columns("users") @@ -275,26 +436,26 @@ indexes = await backend.introspector.list_indexes("users") 内省结果默认会被缓存以提高性能： ```python -# 清除所有内省缓存 +# Clear all introspection cache backend.introspector.clear_cache() -# 使特定作用域的缓存失效 +# Invalidate specific scope from rhosocial.activerecord.backend.introspection.types import IntrospectionScope -# 使所有表相关缓存失效 +# Invalidate all table-related cache backend.introspector.invalidate_cache(scope=IntrospectionScope.TABLE) -# 使特定表的缓存失效 +# Invalidate cache for a specific table backend.introspector.invalidate_cache( scope=IntrospectionScope.TABLE, name="users" ) -# 使特定 schema 的缓存失效 +# Invalidate cache for a specific schema backend.introspector.invalidate_cache( scope=IntrospectionScope.TABLE, name="users", - table_name="users" # 用于限定 schema + table_name="users" # Used to qualify schema ) ``` @@ -316,13 +477,13 @@ backend.introspector.invalidate_cache( 内省操作涉及复杂的 `pg_catalog` 查询，建议利用缓存： ```python -# 首次查询会缓存结果 +# First query caches the result tables = backend.introspector.list_tables() -# 后续查询直接从缓存返回 +# Subsequent queries return from cache tables_again = backend.introspector.list_tables() -# 只有在表结构变更后才需要清除缓存 +# Only clear cache when table structure changes backend.introspector.invalidate_cache(scope=IntrospectionScope.TABLE, name="users") ``` @@ -331,7 +492,7 @@ backend.introspector.invalidate_cache(scope=IntrospectionScope.TABLE, name="user 在多 schema 环境中，明确指定 schema： ```python -# 明确指定 schema 避免歧义 +# Explicitly specify schema to avoid ambiguity table_info = backend.introspector.get_table_info("users", schema="auth") ``` @@ -345,15 +506,15 @@ from rhosocial.activerecord.backend.introspection.types import IndexType indexes = backend.introspector.list_indexes("documents") for idx in indexes: if idx.index_type == IndexType.GIN: - # GIN 索引适合 JSONB 或全文搜索查询 - print(f"GIN 索引可用: {idx.name}") + # GIN indexes are suitable for JSONB or full-text search queries + print(f"GIN index available: {idx.name}") ``` ### 4. 异步环境中的并发查询 ```python async def get_schema_info(backend): - # 并发获取多个表的列信息 + # Concurrently fetch columns for multiple tables import asyncio tables = await backend.introspector.list_tables() tasks = [ @@ -389,6 +550,31 @@ async def get_schema_info(backend): | `clear_cache()` | 清除缓存 | 无 | | `invalidate_cache(scope, ...)` | 使缓存失效 | `scope`, `name`, `table_name` | +### PostgreSQL 专有方法 + +| 方法 | 说明 | 参数 | +|------|------|------| +| `list_materialized_views()` | 列出物化视图 | `schema`, `include_system` | +| `get_materialized_view_info(name)` | 获取单个物化视图 | `name`, `schema` | +| `materialized_view_exists(name)` | 检查物化视图是否存在 | `name`, `schema` | +| `list_sequences()` | 列出序列 | `schema` | +| `get_sequence_info(name)` | 获取单个序列 | `name`, `schema` | +| `sequence_exists(name)` | 检查序列是否存在 | `name`, `schema` | +| `list_extensions()` | 列出已安装扩展 | 无 | +| `get_extension_info(name)` | 获取单个已安装扩展 | `name` | +| `extension_exists(name)` | 检查扩展是否已安装 | `name` | +| `list_unused_indexes()` | 自统计重置以来没有扫描的索引 | `schema` | +| `list_table_usage_stats()` | 逐表使用计数 | `schema` | +| `list_runtime_metrics()` | 每个后端进程一行 | `schema` | +| `list_index_usage_stats()` | 逐索引计数与大小 | `schema` | +| `list_object_dependencies()` | 依赖边，或单个对象的边 | `object_name`, `schema` | +| `get_ddl_statement(name)` | DDL 文本，或本后端上的 `None` | `name`, `object_type`, `schema` | +| `list_column_data_profiles(table_name)` | 来自 `pg_stats` 的逐列剖析 | `table_name`, `schema` | +| `sample_column_values(table_name, column_name)` | 保留的值与频率 | `table_name`, `column_name`, `limit`, `schema` | +| `estimate_join_coverage(...)` | 两个覆盖率分数 | `left_table`, `left_column`, `right_table`, `right_column`, `schema` | + +三个数据方法也声明在核心的内省器上并使用相同名字；PostgreSQL 读取器提供解析，异步孪生类以相同名字 await。 + ## 命令行内省命令 PostgreSQL 后端提供命令行内省命令，无需编写代码即可查询数据库元数据。 @@ -396,19 +582,19 @@ PostgreSQL 后端提供命令行内省命令，无需编写代码即可查询数 ### 基本用法 ```bash -# 列出所有表 +# List all tables python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ --host localhost --port 5432 --database mydb --user postgres --password -# 列出所有视图 +# List all views python -m rhosocial.activerecord.backend.impl.postgres introspect views \ --database mydb --user postgres -# 获取数据库信息 +# Get database information python -m rhosocial.activerecord.backend.impl.postgres introspect database \ --database mydb --user postgres -# 包含系统表 +# Include system tables python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ --database mydb --user postgres --include-system ``` @@ -428,23 +614,23 @@ python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ ### 查询表详情 ```bash -# 仅查询列信息 +# Query only column information python -m rhosocial.activerecord.backend.impl.postgres introspect columns users \ --database mydb --user postgres -# 仅查询索引信息 +# Query only index information python -m rhosocial.activerecord.backend.impl.postgres introspect indexes users \ --database mydb --user postgres -# 仅查询外键信息 +# Query only foreign key information python -m rhosocial.activerecord.backend.impl.postgres introspect foreign-keys posts \ --database mydb --user postgres -# 查询触发器 +# Query triggers python -m rhosocial.activerecord.backend.impl.postgres introspect triggers \ --database mydb --user postgres -# 查询特定表的触发器 +# Query triggers for a specific table python -m rhosocial.activerecord.backend.impl.postgres introspect triggers users \ --database mydb --user postgres ``` @@ -454,16 +640,59 @@ python -m rhosocial.activerecord.backend.impl.postgres introspect triggers users PostgreSQL 支持 schema 参数： ```bash -# 指定 schema 查询表 +# Specify schema to query tables python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ --database mydb --user postgres --schema auth ``` +### 查询运维与数据族 + +六个运维族和三个数据族都是独立的类型： + +```bash +# Indexes no query has used since the statistics were reset +python -m rhosocial.activerecord.backend.impl.postgres introspect unused-indexes \ + --database mydb --user postgres + +# Per-table usage counters and per-index counters +python -m rhosocial.activerecord.backend.impl.postgres introspect table-stats \ + --database mydb --user postgres +python -m rhosocial.activerecord.backend.impl.postgres introspect index-stats \ + --database mydb --user postgres + +# What the server has been doing +python -m rhosocial.activerecord.backend.impl.postgres introspect runtime \ + --database mydb --user postgres + +# The dependency graph, or one object's edges +python -m rhosocial.activerecord.backend.impl.postgres introspect dependencies orders \ + --database mydb --user postgres + +# The DDL that recreates a table (this backend answers "no DDL available") +python -m rhosocial.activerecord.backend.impl.postgres introspect ddl orders \ + --database mydb --user postgres + +# Profiles and samples from pg_stats +python -m rhosocial.activerecord.backend.impl.postgres introspect column-profiles orders \ + --database mydb --user postgres +python -m rhosocial.activerecord.backend.impl.postgres introspect value-samples orders \ + --column status --database mydb --user postgres + +# Overlap between two columns' value domains +python -m rhosocial.activerecord.backend.impl.postgres introspect join-coverage orders \ + --column buyer_id --right-table users --right-column id \ + --database mydb --user postgres + +# One table's storage split +python -m rhosocial.activerecord.backend.impl.postgres introspect table-storage orders \ + --database mydb --user postgres +``` + ### 内省类型 | 类型 | 说明 | NAME | |------|------|------| -| `tables` | 列出所有基础表 | 不接受 | +| `tables` | schema 中的每种关系：基础表、视图、物化视图、外部表与分区表 | 不接受 | | `views` | 列出普通视图，或按名称查询一个视图 | 可选：视图名 | | `columns` | 一张表的列信息 | 必填：表名 | | `indexes` | 一张表的索引信息 | 必填：表名 | @@ -473,13 +702,24 @@ python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ | `extensions` | 已安装的扩展，或按名称查询一个扩展 | 可选：扩展名 | | `materialized-views` | 物化视图，或按名称查询一个 | 可选：视图名 | | `sequences` | 序列，或按名称查询一个 | 可选：序列名 | +| `table-storage` | 一张表的存储拆分与行估计 | 必填：表名 | +| `unused-indexes` | 自统计重置以来 `idx_scan = 0` 的索引 | 不接受 | +| `table-stats` | 逐表使用计数 | 不接受 | +| `runtime` | 来自 `pg_stat_activity` 的每个后端进程一行 | 不接受 | +| `index-stats` | 逐索引计数与大小 | 不接受 | +| `dependencies` | 依赖边，或单个对象的边 | 可选：对象名 | +| `ddl` | 重建一张表的 DDL | 必填：表名 | +| `column-profiles` | 来自 `pg_stats` 的逐列剖析 | 必填：表名 | +| `value-samples` | 一列保留的值与频率 | 必填：表名；`--column` 必填 | +| `join-coverage` | 两列值域的重叠 | 必填：左表；`--column`、`--right-table`、`--right-column` 必填 | + +这里的十个类型是本页此前列出的十个之外新增的；其中九个就是上文那些族。`table-storage` 向声明层询问一张表的存储拆分；`unused-indexes`、`table-stats`、`runtime`、`index-stats`、`dependencies` 与 `ddl` 是六个运维族；`column-profiles`、`value-samples` 与 `join-coverage` 是三个数据族。每个类型都被提供，因为 `_TYPE_CAPABILITIES` 中命名的探针开关在适配后的服务器上回答 `True`；开关为 `False` 的类型会以 `2` 退出并提示 "not supported by this server"，而不是返回空列表。 + +`NAME` 指明该类型要回答的对象，每种类型对“是否接受名称”只有一个答案。`columns`、`indexes`、`foreign-keys`、`table-storage`、`ddl`、`column-profiles`、`value-samples` 与 `join-coverage` 的名称是它们所涉及的表，必须给出——对 `join-coverage` 而言是左表。`triggers` 接受表名，`dependencies` 接受对象名，`views`、`materialized-views`、`sequences`、`extensions` 接受对象名；不给名称时各自列出整族。`tables`、`database`、`unused-indexes`、`table-stats`、`runtime` 与 `index-stats` 不接受名称：在那里给名称是错误而不是过滤条件，因为这些计数列表回答的是连接的可见范围，而不是单个对象。 + +有三个类型通过自己的标志询问一列或第二张表。`value-samples` 接受 `--column`，`join-coverage` 接受 `--column`、`--right-table` 与 `--right-column`；缺少其中一个是未完成的调用，会以 `1` 退出。把标志给一个对它没有问题可问的类型，会以 `2` 退出而不是被忽略，规则与 NAME 相同。 -`NAME` 指明该类型要回答的对象，每种类型对"是否接受名称"只有一个答案。 -`columns`、`indexes`、`foreign-keys` 的名称是它们所属的表，必须给出； -`triggers` 接受表名，不给则列出全部触发器；`views`、`materialized-views`、 -`sequences`、`extensions` 接受对象名，不给则列出整族。`tables` 与 `database` -不接受名称：在那里给名称是错误而不是过滤条件，因此针对单个对象的问题不会 -悄悄返回整族列表。 +`ddl` 是唯一一个可以在一张存在的表上合理报告失败的类型：本后端的投影不携带 DDL 文本，所以 `get_ddl_statement` 返回 `None`，命令会打印 `Error: no DDL available for ''` 并以 `1` 退出。它不会打印空行，因为一行没有文本的记录会被读成一个答案。这里 `supports_ddl_extraction_native` 为 `False`，所以需要服务器原生文本的调用方事先就知道答案是组装，而不是获取。 运行以 `1` 退出表示没能完成——连接失败、查询失败，或名称所指的对象不存在； 以 `2` 退出表示这个请求在此不成立：未知类型、该类型不接受名称，或本服务器 @@ -488,19 +728,19 @@ python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ ### 输出格式 ```bash -# 表格格式（默认，需要 rich 库） +# Table format (default, requires rich library) python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ --database mydb --user postgres -# JSON 格式 +# JSON format python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ --database mydb --user postgres --output json -# CSV 格式 +# CSV format python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ --database mydb --user postgres --output csv -# TSV 格式 +# TSV format python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ --database mydb --user postgres --output tsv ``` @@ -508,7 +748,7 @@ python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ ### 使用异步后端 ```bash -# 使用 --use-async 参数启用异步模式 +# Use --use-async flag to enable async mode python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ --database mydb --user postgres --use-async ``` @@ -518,13 +758,13 @@ python -m rhosocial.activerecord.backend.impl.postgres introspect tables \ 可以在环境中设置连接参数，简化命令行调用： ```bash -# 设置环境变量 +# Set environment variables export POSTGRES_HOST=localhost export POSTGRES_PORT=5432 export POSTGRES_DATABASE=mydb export POSTGRES_USER=postgres export POSTGRES_PASSWORD=secret -# 直接使用命令 +# Use command directly python -m rhosocial.activerecord.backend.impl.postgres introspect tables ``` From 0322fbb532057562d3aac5631c9bc88988873e99 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 8 Nov 2026 13:24:13 +0800 Subject: [PATCH 25/27] docs: state what PostgreSQL answers, including what pg_stats cannot give This corrects the page the earlier commit added. The operational section now says five of the six families read the statistics collector and that DDL is the exception reading pg_catalog, rather than generalising across all six. The data section now separates the two pg_stats-backed families from join coverage, which reads the rows of both tables because pg_stats keeps only a bounded sample. The CLI note now says the advertised type list is static and the capability switch is asked after the dialect has adapted, rather than implying each type is offered because its switch answers True. The stale version table, which claimed full partition information support against a probe that answers False, is replaced by the one version gate in this code: pg_sequences is read only on PostgreSQL 10 and later, and an unknown version takes the narrower form. --- .../introspection.md | 55 +++++++++++-------- .../introspection.md | 15 ++--- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/docs/en_US/postgres_specific_features/introspection.md b/docs/en_US/postgres_specific_features/introspection.md index 349c35bf..e480f3ab 100644 --- a/docs/en_US/postgres_specific_features/introspection.md +++ b/docs/en_US/postgres_specific_features/introspection.md @@ -327,12 +327,14 @@ for idx in indexes: ## Operational Introspection -The declaration layer answers what the server *is*. The six families here answer -what it has *been doing*, and they read the statistics collector's views rather -than `pg_catalog` proper. The tense is PostgreSQL's: every counter is since the +The declaration layer answers what the server *is*. Five of the six families +here answer what it has *been doing*, and they read the statistics collector's +views rather than `pg_catalog` proper. The sixth, DDL, is the exception in both +directions: it reads `pg_catalog`, and what it answers is what an object was +declared as. The tense for the five is PostgreSQL's: every counter is since the statistics were last reset — a crash or a `pg_stat_reset()` — so a zero means -nothing happened in the window, never nothing ever. A number without that -sentence is not a fact, which is why the tense is stated here for every family. +nothing happened in the window, never nothing ever. A counter without that +sentence is not a fact, which is why the tense is stated for each of the five. | Method | Reads | Returns | |--------|-------|---------| @@ -424,12 +426,16 @@ an empty row, because a row with no text would read as an answer. ## Column Data Profiles -The three data families read `pg_stats`, the per-column statistics `ANALYZE` -leaves, not the rows themselves. `pg_stats` holds one row per analysed column, -so the profile costs one catalogue query however large the table is — a catalogue -read rather than a pass over the rows, after the cheap column listing core reads -first. The price is that every figure is an estimate of the last `ANALYZE`, which -is why the answer carries its source and the time it was collected. +Two of the three data families read `pg_stats`, the per-column statistics +`ANALYZE` leaves, not the rows themselves: the profile and the value sample. +`pg_stats` holds one row per analysed column, so the profile costs one catalogue +query however large the table is — a catalogue read rather than a pass over the +rows, after the cheap column listing core reads first. The price is that every +figure is an estimate of the last `ANALYZE`, which is why the answer carries its +source and the time it was collected. Join coverage is the exception: it reads +the rows of both tables, because `pg_stats` keeps only a bounded sample of each +column, and an overlap computed from that sample would be a guess dressed as a +measurement. | Method | Reads | Returns | |--------|-------|---------| @@ -637,14 +643,17 @@ backend.introspector.invalidate_cache( ## Version Compatibility -Introspection behavior differences across PostgreSQL versions: +One answer on this page depends on the server's version. `pg_sequences` arrived +in PostgreSQL 10, so `list_sequences()` reads it only when `server_version` is +at least `(10, 0, 0)`; on an older server the statement omits the view and +`cache_size` and `last_value` are absent, which is "not measured" rather than +zero. An unknown version takes the narrower form too: the introspector does not +assume a server it has not identified carries the newer catalogue. -| Feature | PostgreSQL 12+ | PostgreSQL 11 and below | -|---------|----------------|------------------------| -| Materialized view info | Full support | Basic support | -| Partitioned table info | Full support | Limited support | -| Generated columns | Supported | Not supported | -| Stored procedure triggers | Supported | Not supported | +Every other statement on this page has no version gate. `supports_partition_info` +is `False` at every version, not only on old ones: what the version differences +change is partition DDL, which is a separate protocol family, not the listing +that labels a partitioned table. ## Best Practices @@ -895,10 +904,12 @@ Ten rows here are new relative to the ten types this page listed before; nine of them are the families above. `table-storage` asks the declaration layer for one table's storage split; `unused-indexes`, `table-stats`, `runtime`, `index-stats`, `dependencies` and `ddl` are the six operational families; `column-profiles`, -`value-samples` and `join-coverage` are the three data families. Each is offered -because the probe switch named in `_TYPE_CAPABILITIES` answers `True` on the -adapted server; a type whose switch is `False` exits `2` with "not supported by -this server" rather than returning an empty list. +`value-samples` and `join-coverage` are the three data families. Each type names, +in `_TYPE_CAPABILITIES`, the probe switch that decides whether the connected +server can answer it; the advertised list itself is static, and the switch is +asked after the dialect has been adapted to the server. A type whose switch is +`False` exits `2` with "not supported by this server" rather than returning an +empty list. `NAME` selects the object the type answers for, and each type has exactly one answer to whether it takes one. `columns`, `indexes`, `foreign-keys`, diff --git a/docs/zh_CN/postgres_specific_features/introspection.md b/docs/zh_CN/postgres_specific_features/introspection.md index 83b06ed8..a210fc7f 100644 --- a/docs/zh_CN/postgres_specific_features/introspection.md +++ b/docs/zh_CN/postgres_specific_features/introspection.md @@ -296,7 +296,7 @@ for idx in indexes: ## 运维内省 -声明层回答服务器*是什么*。这里的六个族回答它*一直在做什么*，它们读取统计收集器的视图，而不是 `pg_catalog` 本身。时态是 PostgreSQL 的：每个计数都是自统计上次被重置以来——一次崩溃或一次 `pg_stat_reset()`——所以零表示窗口内什么都没发生，永远不是“从来没有”。一个没有这句话的数字不是事实，因此这里为每个族都写明时态。 +声明层回答服务器*是什么*。这里的六个族中有五个回答它*一直在做什么*，它们读取统计收集器的视图，而不是 `pg_catalog` 本身。第六个是 DDL，它在两方面都是例外：它读取 `pg_catalog`，回答的是一个对象被声明成了什么。这五个族的时态是 PostgreSQL 的：每个计数都是自统计上次被重置以来——一次崩溃或一次 `pg_stat_reset()`——所以零表示窗口内什么都没发生，永远不是“从来没有”。一个没有这句话的计数不是事实，因此这里为这五个族逐一写明时态。 | 方法 | 读取 | 返回 | |------|------|------| @@ -335,7 +335,7 @@ PostgreSQL 没有为未使用索引提供厂商视图，所以该列表由计数 ## 列数据剖析 -三个数据族读取 `pg_stats`，也就是 `ANALYZE` 留下的逐列统计，而不是行本身。`pg_stats` 每个已分析列一行，所以无论表多大，剖析本身都只花一条目录查询——这是一次目录读取，而不是对行的遍历，核心在此之前还会先读一次廉价的列清单。代价是每个数字都是上一次 `ANALYZE` 的估计，这也是答案要携带来源与收集时间的原因。 +三个数据族中有两个读取 `pg_stats`，也就是 `ANALYZE` 留下的逐列统计，而不是行本身：剖析与值采样。`pg_stats` 每个已分析列一行，所以无论表多大，剖析本身都只花一条目录查询——这是一次目录读取，而不是对行的遍历，核心在此之前还会先读一次廉价的列清单。代价是每个数字都是上一次 `ANALYZE` 的估计，这也是答案要携带来源与收集时间的原因。连接覆盖率是例外：它读取两张表的行，因为 `pg_stats` 只保留每列的一个有界样本，用该样本算出的重叠只会是披着测量的猜测。 | 方法 | 读取 | 返回 | |------|------|------| @@ -461,14 +461,9 @@ backend.introspector.invalidate_cache( ## 版本兼容性 -不同 PostgreSQL 版本的内省行为差异： +本页上只有一个内省答案取决于服务器版本。`pg_sequences` 在 PostgreSQL 10 才出现，所以 `list_sequences()` 仅在 `server_version` 至少为 `(10, 0, 0)` 时才读取它；在更旧的服务器上，语句会略去该视图，`cache_size` 与 `last_value` 缺席，这是“未测量”而不是零。版本未知时同样采用较窄的形式：内省器不会假定一个它尚未识别的服务器带有更新的目录。 -| 功能 | PostgreSQL 12+ | PostgreSQL 11 及以下 | -|------|----------------|---------------------| -| 物化视图信息 | 完整支持 | 基础支持 | -| 分区表信息 | 完整支持 | 有限支持 | -| 生成列 | 支持 | 不支持 | -| 存储过程触发器 | 支持 | 不支持 | +本页其余语句都没有版本门控。`supports_partition_info` 在任何版本上都是 `False`，不只是旧版本：版本差异改变的是分区 DDL，那是另一个协议族，而不是给分区表打标签的内省列表。 ## 最佳实践 @@ -713,7 +708,7 @@ python -m rhosocial.activerecord.backend.impl.postgres introspect table-storage | `value-samples` | 一列保留的值与频率 | 必填：表名；`--column` 必填 | | `join-coverage` | 两列值域的重叠 | 必填：左表；`--column`、`--right-table`、`--right-column` 必填 | -这里的十个类型是本页此前列出的十个之外新增的；其中九个就是上文那些族。`table-storage` 向声明层询问一张表的存储拆分；`unused-indexes`、`table-stats`、`runtime`、`index-stats`、`dependencies` 与 `ddl` 是六个运维族；`column-profiles`、`value-samples` 与 `join-coverage` 是三个数据族。每个类型都被提供，因为 `_TYPE_CAPABILITIES` 中命名的探针开关在适配后的服务器上回答 `True`；开关为 `False` 的类型会以 `2` 退出并提示 "not supported by this server"，而不是返回空列表。 +这里的十个类型是本页此前列出的十个之外新增的；其中九个就是上文那些族。`table-storage` 向声明层询问一张表的存储拆分；`unused-indexes`、`table-stats`、`runtime`、`index-stats`、`dependencies` 与 `ddl` 是六个运维族；`column-profiles`、`value-samples` 与 `join-coverage` 是三个数据族。每个类型都在 `_TYPE_CAPABILITIES` 中指明决定连接服务器能否回答它的探针开关；对外列出的类型清单本身是静态的，开关在方言适配到服务器之后才被询问。开关为 `False` 的类型会以 `2` 退出并提示 "not supported by this server"，而不是返回空列表。 `NAME` 指明该类型要回答的对象，每种类型对“是否接受名称”只有一个答案。`columns`、`indexes`、`foreign-keys`、`table-storage`、`ddl`、`column-profiles`、`value-samples` 与 `join-coverage` 的名称是它们所涉及的表，必须给出——对 `join-coverage` 而言是左表。`triggers` 接受表名，`dependencies` 接受对象名，`views`、`materialized-views`、`sequences`、`extensions` 接受对象名；不给名称时各自列出整族。`tables`、`database`、`unused-indexes`、`table-stats`、`runtime` 与 `index-stats` 不接受名称：在那里给名称是错误而不是过滤条件，因为这些计数列表回答的是连接的可见范围，而不是单个对象。 From bce17e451a4c90aff91f754850b6ad4f1120c9d7 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 8 Nov 2026 14:57:34 +0800 Subject: [PATCH 26/27] ci: pin core to release/v1.0.0.dev30 --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c93f77e7..a4c72039 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Install dependencies @@ -278,7 +278,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Install dependencies @@ -385,7 +385,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Install dependencies @@ -488,7 +488,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/introspection-enhancement" + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Install dependencies From 351661251bf99f393aba13854267dbf85e3a40c8 Mon Sep 17 00:00:00 2001 From: zhao vistart Date: Sun, 8 Nov 2026 15:16:42 +0800 Subject: [PATCH 27/27] chore: add changelog fragments for PR #64 --- changelog.d/64.added.md | 1 + changelog.d/64.removed.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/64.added.md create mode 100644 changelog.d/64.removed.md diff --git a/changelog.d/64.added.md b/changelog.d/64.added.md new file mode 100644 index 00000000..86465754 --- /dev/null +++ b/changelog.d/64.added.md @@ -0,0 +1 @@ +Own the status families core moved out -- `WALInfo`, `ReplicationInfo`, `ArchiveInfo` and `SecurityInfo` -- beside `PostgreSQLServerOverview`, keeping `get_overview()`'s signature through a covariant return. Render `pg_relation_size` and `pg_indexes_size` as expression nodes bound to the listing row's oid so the total rides the table listing; a row count prefers `reltuples`, falls back to `n_live_tup` and normalizes `-1` to `None`; `table-storage` becomes a CLI type because the capability had to be askable. Answer the operational families from `pg_stat_user_indexes`, `pg_stat_user_tables`, `pg_stat_activity` and `pg_depend` (`pg_stat_statements` is deliberately not read), and profile columns and samples from `pg_stats`, labelled `ESTIMATE` or `UNKNOWN` by vintage, with arrays through `to_jsonb` and exact join coverage. DDL extraction is declared but answers without text: `get_ddl_statement` returns `None`, the CLI exits 1 with "no DDL available", and `supports_ddl_extraction_native()` is `False`. Ten CLI types derive from `_TYPE_CAPABILITIES`, `supports_extensions` delegates so one capability is one answer, and `async_support.py` supplies `run_coroutine()` for the CLI's three `--async` modes on Windows. diff --git a/changelog.d/64.removed.md b/changelog.d/64.removed.md new file mode 100644 index 00000000..253b5f9c --- /dev/null +++ b/changelog.d/64.removed.md @@ -0,0 +1 @@ +Rename `SyncPostgreSQLIntrospector` to `PostgreSQLIntrospector` and `SyncPostgreSQLStatusIntrospector` to `PostgreSQLStatusIntrospector`. Backend developers subclassing the old names must attach to core's `AbstractIntrospector` and `AbstractStatusIntrospector` through `IntrospectorExecutor`.