diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b9d00012..7cf8d6ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" # NOSONAR + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/partition-maintenance" # NOSONAR pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Install dependencies @@ -278,7 +278,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" # NOSONAR + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/partition-maintenance" # NOSONAR pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Install dependencies @@ -385,7 +385,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" # NOSONAR + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/partition-maintenance" # NOSONAR pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Install dependencies @@ -488,7 +488,7 @@ jobs: - name: Install activerecord and testsuite run: | - pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@release/v1.0.0.dev30" # NOSONAR + pip install "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/partition-maintenance" # NOSONAR pip install "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@release/v1.0.0.dev17" - name: Install dependencies diff --git a/changelog.d/67.added.md b/changelog.d/67.added.md new file mode 100644 index 00000000..7a04dc0a --- /dev/null +++ b/changelog.d/67.added.md @@ -0,0 +1 @@ +Added PostgreSQL partition planning and maintenance helpers: idempotent nsure, recursive list, alidate, drop_before and explicit drop_branch, with sync and async executors. diff --git a/docs/en_US/postgres_specific_features/partition.md b/docs/en_US/postgres_specific_features/partition.md index fd1125af..bdb310d6 100644 --- a/docs/en_US/postgres_specific_features/partition.md +++ b/docs/en_US/postgres_specific_features/partition.md @@ -19,10 +19,98 @@ from rhosocial.activerecord.backend.impl.postgres.expression.ddl.partition impor partition = PostgresCreatePartitionStatement( dialect, parent_table="orders", partition_name="orders_2024_q1", - partition_type="RANGE", bounds="FROM ('2024-01-01') TO ('2024-04-01')", + partition_type="RANGE", + partition_values={"from": "2024-01-01", "to": "2024-04-01"}, ) ``` +## Nested Partitions + +PostgreSQL lets a partition be partitioned again, and it lets each branch of a +HASH parent pick its own child strategy. Pass a `PartitionClause` to +`PostgresCreatePartitionStatement`; the rendered order is `FOR VALUES` then +`PARTITION BY`. + +```python +from rhosocial.activerecord.backend.expression import Column +from rhosocial.activerecord.backend.expression.statements import ( + PartitionClause, PartitionStrategy, +) + +# CREATE TABLE logs_p0 PARTITION OF logs +# FOR VALUES WITH (MODULUS 4, REMAINDER 0) +# PARTITION BY RANGE (created_at) +branch = PostgresCreatePartitionStatement( + dialect, parent_table="logs", partition_name="logs_p0", + partition_type="HASH", partition_values={"modulus": 4, "remainder": 0}, + partition_clause=PartitionClause( + dialect, PartitionStrategy.RANGE, [Column(dialect, "created_at")], + ), +) + +# CREATE TABLE logs_p0_2026_01 PARTITION OF logs_p0 +# FOR VALUES FROM ('2026-01-01 00:00:00+00') TO ('2026-02-01 00:00:00+00') +leaf = PostgresCreatePartitionStatement( + dialect, parent_table="logs_p0", partition_name="logs_p0_2026_01", + partition_type="RANGE", + partition_values={"from": "2026-01-01 00:00:00+00", + "to": "2026-02-01 00:00:00+00"}, +) +``` + +A parent primary key must contain every level's partition key. With a HASH +parent on `tenant_bucket` and RANGE children on `created_at`, the parent key is +`(id, tenant_bucket, created_at)`; PostgreSQL rejects a child that is itself +partitioned when the inherited key omits `created_at`. + +## Partition Maintenance + +`PostgresPartitionPlan` describes the tree without naming individual +partitions; `PostgresPartitionMaintenance` (and its async twin) reconciles it +with the catalog. The module is owner-agnostic: table names and retention +decisions stay with the application. + +```python +from datetime import datetime, timezone +from rhosocial.activerecord.backend.impl.postgres.partition import ( + HashBranch, PostgresPartitionMaintenance, PostgresPartitionPlan, RangePolicy, +) + +start = datetime(2026, 1, 1, tzinfo=timezone.utc) +plan = PostgresPartitionPlan( + parent="logs", method="HASH", columns=("tenant_bucket",), modulus=4, + branches=( + HashBranch(0, RangePolicy(columns=("created_at",), start=start, + interval="month", count=12)), + HashBranch(1, RangePolicy(columns=("created_at",), start=start, + interval="quarter", count=4)), + HashBranch(2, RangePolicy(columns=("created_at",), start=start, + interval="year", count=2)), + HashBranch(3, RangePolicy(columns=("created_at",), + bounds=((start, start.replace(day=16)),))), + ), +) + +maintenance = PostgresPartitionMaintenance(backend) +report = maintenance.ensure(plan) # idempotent gap filling +report.missing # (parent, child) pairs +maintenance.list(plan).orphans # catalog-only partitions +maintenance.validate(plan).issues # gaps / overlaps / mismatches +maintenance.drop_before(plan, cutoff, dry_run=True)# per-branch retention +``` + +* `ensure` creates only missing partitions. PostgreSQL DDL is transactional, so + each unit (one branch with its leaves, or all leaves of a flat plan) commits + independently. +* `list` reads `pg_inherits`/`pg_class` recursively and keeps the server's + bound text verbatim. +* `drop_before` only drops true leaves; a branch that is itself partitioned is + never cascaded. It defaults to `dry_run=True`. +* `drop_branch(plan, plan.branches[0])` is the explicit whole-branch removal. +* A flat plan (`policy=RangePolicy(...)` with no `branches`) produces the + single-level shape used by monthly log tables. + + ## Partition Management PostgreSQL does not support `ATTACH PARTITION CONCURRENTLY`. PostgreSQL 14+ supports @@ -36,7 +124,8 @@ from rhosocial.activerecord.backend.impl.postgres.expression.ddl.partition impor # ATTACH PARTITION attach = PostgresAttachPartitionStatement( dialect, parent_table="orders", partition_name="orders_new", - partition_type="RANGE", bounds="FROM ('2024-07-01') TO ('2024-10-01')", + partition_type="RANGE", + partition_values={"from": "2024-07-01", "to": "2024-10-01"}, ) # DETACH PARTITION CONCURRENTLY (PG 14+) diff --git a/docs/zh_CN/postgres_specific_features/partition.md b/docs/zh_CN/postgres_specific_features/partition.md index 3b84bc72..f0d9df5d 100644 --- a/docs/zh_CN/postgres_specific_features/partition.md +++ b/docs/zh_CN/postgres_specific_features/partition.md @@ -23,11 +23,93 @@ partition = PostgresCreatePartitionStatement( parent_table="orders", partition_name="orders_2024_q1", partition_type="RANGE", - bounds="FROM ('2024-01-01') TO ('2024-04-01')", + partition_values={"from": "2024-01-01", "to": "2024-04-01"}, ) # sql: 'CREATE TABLE "orders_2024_q1" PARTITION OF "orders" FOR VALUES FROM ('2024-01-01') TO ('2024-04-01')' ``` +## 多级（子）分区 + +PostgreSQL 允许一个分区继续分区，且 HASH 父表的每个一级分支可以各自选择 +二级策略。把 `PartitionClause` 传给 `PostgresCreatePartitionStatement`， +渲染顺序为 `FOR VALUES` → `PARTITION BY`。 + +```python +from rhosocial.activerecord.backend.expression import Column +from rhosocial.activerecord.backend.expression.statements import ( + PartitionClause, PartitionStrategy, +) + +# CREATE TABLE logs_p0 PARTITION OF logs +# FOR VALUES WITH (MODULUS 4, REMAINDER 0) +# PARTITION BY RANGE (created_at) +branch = PostgresCreatePartitionStatement( + dialect, parent_table="logs", partition_name="logs_p0", + partition_type="HASH", partition_values={"modulus": 4, "remainder": 0}, + partition_clause=PartitionClause( + dialect, PartitionStrategy.RANGE, [Column(dialect, "created_at")], + ), +) + +# CREATE TABLE logs_p0_2026_01 PARTITION OF logs_p0 +# FOR VALUES FROM ('2026-01-01 00:00:00+00') TO ('2026-02-01 00:00:00+00') +leaf = PostgresCreatePartitionStatement( + dialect, parent_table="logs_p0", partition_name="logs_p0_2026_01", + partition_type="RANGE", + partition_values={"from": "2026-01-01 00:00:00+00", + "to": "2026-02-01 00:00:00+00"}, +) +``` + +父表主键必须包含**每一级**的分区键。若一级按 `tenant_bucket` HASH、二级按 +`created_at` RANGE，父表主键应为 `(id, tenant_bucket, created_at)`；当继承的 +唯一键缺少 `created_at` 时，PostgreSQL 会拒绝“本身又分区”的一级子表。 + +## 分区维护 + +`PostgresPartitionPlan` 只描述树（父表、分区键、分支策略、命名），不写死具体 +分区名；`PostgresPartitionMaintenance`（及其异步版本）把它与目录状态对齐。 +模块不绑定任何业务表名或保留策略，这些仍由应用决定。 + +```python +from datetime import datetime, timezone +from rhosocial.activerecord.backend.impl.postgres.partition import ( + HashBranch, PostgresPartitionMaintenance, PostgresPartitionPlan, RangePolicy, +) + +start = datetime(2026, 1, 1, tzinfo=timezone.utc) +plan = PostgresPartitionPlan( + parent="logs", method="HASH", columns=("tenant_bucket",), modulus=4, + branches=( + HashBranch(0, RangePolicy(columns=("created_at",), start=start, + interval="month", count=12)), + HashBranch(1, RangePolicy(columns=("created_at",), start=start, + interval="quarter", count=4)), + HashBranch(2, RangePolicy(columns=("created_at",), start=start, + interval="year", count=2)), + HashBranch(3, RangePolicy(columns=("created_at",), + bounds=((start, start.replace(day=16)),))), + ), +) + +maintenance = PostgresPartitionMaintenance(backend) +report = maintenance.ensure(plan) # 幂等补建缺失分区 +report.missing # (父表, 缺失分区) 列表 +maintenance.list(plan).orphans # 计划外的分区 +maintenance.validate(plan).issues # 缺口 / 重叠 / 策略不符 +maintenance.drop_before(plan, cutoff, dry_run=True) # 按分支策略回收 +``` + +* `ensure` 只补建缺失项；PostgreSQL 的 DDL 是事务性的，每个单元（一个一级 + 分支及其叶子，或单层计划的全部叶子）独立提交。 +* `list` 递归读取 `pg_inherits` / `pg_class`，并原样保留服务器给出的 bound 文本。 +* `drop_before` 只删除真正的叶子；自身已分区的一级分支不会被级联删除，默认 + `dry_run=True`。 +* `drop_branch(plan, plan.branches[0])` 是删除整个一级分支的显式操作。 +* 单层计划（只给 `policy=RangePolicy(...)`、不给 `branches`）生成按月日志表 + 所需的扁平形态。 + + ## 分区管理 PostgreSQL 不支持 `ATTACH PARTITION CONCURRENTLY`。PostgreSQL 14+ 支持 @@ -43,7 +125,8 @@ from rhosocial.activerecord.backend.impl.postgres.expression.ddl.partition impor # 附加分区 attach = PostgresAttachPartitionStatement( dialect, parent_table="orders", partition_name="orders_new", - partition_type="RANGE", bounds="FROM ('2024-07-01') TO ('2024-10-01')", + partition_type="RANGE", + partition_values={"from": "2024-07-01", "to": "2024-10-01"}, ) # sql: 'ALTER TABLE "orders" ATTACH PARTITION "orders_new" FOR VALUES FROM ('2024-07-01') TO ('2024-10-01')' diff --git a/src/rhosocial/activerecord/backend/impl/postgres/examples/ddl/partition.py b/src/rhosocial/activerecord/backend/impl/postgres/examples/ddl/partition.py index 7f774972..b8e52eaf 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/examples/ddl/partition.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/examples/ddl/partition.py @@ -2,9 +2,9 @@ Table partitioning - PostgreSQL 10+. This example demonstrates: -1. CREATE TABLE ... PARTITION BY RANGE -2. Creating individual partitions -3. Inserting data into partitioned table +1. CREATE TABLE ... PARTITION BY RANGE with expression nodes +2. Creating individual partitions (CREATE TABLE ... PARTITION OF) +3. Inserting data into a partitioned table 4. Querying partitioned tables """ @@ -14,17 +14,25 @@ import os from rhosocial.activerecord.backend.impl.postgres.backend import PostgresBackend from rhosocial.activerecord.backend.impl.postgres.config import PostgresConnectionConfig -from rhosocial.activerecord.backend.impl.postgres.expression.ddl.partition import ( +from rhosocial.activerecord.backend.impl.postgres.expression import ( PostgresCreatePartitionStatement, + PostgresSerialType, ) from rhosocial.activerecord.backend.expression import ( + ColumnDefinition, + ColumnConstraint, + ColumnNotNullConstraint, + CreateTableStatement, DropTableStatement, InsertStatement, + PartitionClause, + PartitionStrategy, ValuesSource, QueryStatement, ) from rhosocial.activerecord.backend.expression.objects import Table from rhosocial.activerecord.backend.expression.core import Literal, AllColumnsWildcard, Column +from rhosocial.activerecord.backend.expression.types import DateType, DecimalType, VarCharType from rhosocial.activerecord.backend.options import ExecutionOptions from rhosocial.activerecord.backend.schema import StatementType @@ -55,17 +63,37 @@ # SECTION: Business Logic (the pattern to learn) # ============================================================ -# 1. Create partitioned parent table using raw SQL -# (Expression API does not yet support PARTITION BY syntax) -parent_sql = """ -CREATE TABLE orders ( - id SERIAL, - order_date DATE NOT NULL, - customer_name VARCHAR(100), - amount NUMERIC(10,2) -) PARTITION BY RANGE (order_date) -""" -backend.execute(parent_sql) +# 1. Create the partitioned parent table with the expression API. +# PARTITION BY is a clause node on CreateTableStatement; the SQL is the +# dialect's, not the caller's. +parent_expr = CreateTableStatement( + dialect=dialect, + table=Table(dialect, 'orders'), + columns=[ + ColumnDefinition( + dialect, 'id', PostgresSerialType(dialect=dialect), + ), + ColumnDefinition( + dialect, 'order_date', DateType(dialect=dialect), + constraints=[ + ColumnConstraint(dialect, branch=ColumnNotNullConstraint(dialect)), + ], + ), + ColumnDefinition( + dialect, 'customer_name', VarCharType(length=100, dialect=dialect), + ), + ColumnDefinition( + dialect, 'amount', DecimalType(precision=10, scale=2, dialect=dialect), + ), + ], + partition=PartitionClause( + dialect=dialect, + method=PartitionStrategy.RANGE, + keys=[Column(dialect, 'order_date')], + ), +) +sql, params = parent_expr.to_sql() +backend.execute(sql, params) print("Created partitioned table: orders") # 2. Create individual partitions using raw SQL @@ -149,8 +177,13 @@ # Key points: # 1. Requires PostgreSQL 10+ for declarative partitioning # 2. PARTITION BY RANGE is the most common strategy -# 3. Other strategies: PARTITION BY LIST, PARTITION BY HASH +# 3. Other strategies: PARTITION BY LIST, PARTITION BY HASH (PG 11+) # 4. PostgreSQL automatically routes INSERT to correct partition # 5. Queries on parent table are transparent across all partitions # 6. Individual partitions can be queried directly for performance -# 7. Expression API does not yet support PARTITION BY - use raw SQL +# 7. Nested partitioning is supported: a partition created with +# PostgresCreatePartitionStatement(partition_clause=...) is itself +# PARTITION BY ; PostgreSQL lets each HASH branch choose its own +# child strategy. See PostgresPartitionPlan for the tree-maintenance API. +# 8. A partitioned table's primary key must include every level's partition +# key, otherwise the server rejects the child that is itself partitioned. diff --git a/src/rhosocial/activerecord/backend/impl/postgres/expression/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/expression/__init__.py index 0c332f00..46742f86 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/expression/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/expression/__init__.py @@ -66,6 +66,7 @@ PostgresDetachPartitionStatement, PostgresAttachPartitionStatement, PostgresPartitionMetadataQueryStatement, + PostgresPartitionTreeQueryStatement, PostgresAlterIndexStatement, PostgresAlterIndexActionType, PostgresReindexStatement, @@ -231,6 +232,7 @@ "PostgresDetachPartitionStatement", "PostgresAttachPartitionStatement", "PostgresPartitionMetadataQueryStatement", + "PostgresPartitionTreeQueryStatement", "PostgresAlterIndexStatement", "PostgresAlterIndexActionType", "PostgresReindexStatement", diff --git a/src/rhosocial/activerecord/backend/impl/postgres/expression/ddl/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/expression/ddl/__init__.py index c0f06e3b..87d6e127 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/expression/ddl/__init__.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/expression/ddl/__init__.py @@ -79,6 +79,7 @@ PostgresDetachPartitionStatement, PostgresAttachPartitionStatement, PostgresPartitionMetadataQueryStatement, + PostgresPartitionTreeQueryStatement, ) from .index import ( PostgresAlterIndexStatement, @@ -225,6 +226,7 @@ "PostgresDetachPartitionStatement", "PostgresAttachPartitionStatement", "PostgresPartitionMetadataQueryStatement", + "PostgresPartitionTreeQueryStatement", # index "PostgresAlterIndexStatement", "PostgresAlterIndexActionType", diff --git a/src/rhosocial/activerecord/backend/impl/postgres/expression/ddl/partition.py b/src/rhosocial/activerecord/backend/impl/postgres/expression/ddl/partition.py index 5a3d809f..ab0620a1 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/expression/ddl/partition.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/expression/ddl/partition.py @@ -34,6 +34,7 @@ "PostgresDetachPartitionStatement", "PostgresAttachPartitionStatement", "PostgresPartitionMetadataQueryStatement", + "PostgresPartitionTreeQueryStatement", ] # ``PARTITION BY {RANGE|LIST|HASH} (...)`` is core's clause: PostgreSQL's @@ -358,4 +359,47 @@ def __init__( @property def format_method(self) -> str: """The dialect formatting method that renders this expression.""" - return "format_partition_metadata_query_statement" \ No newline at end of file + return "format_partition_metadata_query_statement" + + +class PostgresPartitionTreeQueryStatement(BaseExpression): + """PostgreSQL catalog query for the whole partition subtree. + + ``PostgresPartitionMetadataQueryStatement`` answers one level: the parent's + partition key and its direct children. Nested PostgreSQL trees need the + descendants too, and a maintenance report needs the level, the parent name, + the server's bound text, and a size/row estimate. This expression builds + that recursive query; ``PostgresPartitionMixin`` owns the SQL text. + + Attributes: + root_table: Root partitioned table name. + schema: Optional schema; when omitted the search path resolves the + root, exactly as ``to_regclass`` does. + include_stats: Whether to read ``pg_class`` estimates and + ``pg_total_relation_size`` for every node. + """ + + def __init__( + self, + dialect: "SQLDialectBase", + root_table: str, + schema: Optional[str] = None, + *, + include_stats: bool = True, + ): + super().__init__(dialect) + if not root_table: + raise ValueError("root_table must not be empty") + self.root_table = root_table + self.schema = schema + self.include_stats = include_stats + + @property + def qualified_name(self) -> str: + """Return the ``to_regclass`` argument for the root table.""" + return "%s.%s" % (self.schema, self.root_table) if self.schema else self.root_table + + @property + def format_method(self) -> str: + """The dialect formatting method that renders this expression.""" + return "format_partition_tree_query_statement" diff --git a/src/rhosocial/activerecord/backend/impl/postgres/mixins/ddl/partition.py b/src/rhosocial/activerecord/backend/impl/postgres/mixins/ddl/partition.py index 7645e324..b6530102 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/mixins/ddl/partition.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/mixins/ddl/partition.py @@ -20,6 +20,7 @@ PostgresDetachPartitionStatement, PostgresAttachPartitionStatement, PostgresPartitionMetadataQueryStatement, + PostgresPartitionTreeQueryStatement, ) @@ -680,6 +681,65 @@ def format_attach_partition_statement( return " ".join(parts), tuple(params) + def format_partition_tree_query_statement( + self, expr: "PostgresPartitionTreeQueryStatement" + ) -> Tuple[str, tuple]: + """Format the recursive pg_catalog query for a partition subtree. + + The query starts at the root table named by ``expr.qualified_name`` + (resolved with ``to_regclass`` so schema-less callers follow the search + path) and walks ``pg_inherits`` to every descendant. Each row carries + the server's own bound text, the child strategy, the tree level and, + optionally, relation-size estimates. + + Args: + expr: PostgresPartitionTreeQueryStatement instance. + + Returns: + Tuple of (SQL string with one ``%s`` placeholder, params tuple). + """ + if not self.supports_table_partitioning(): + raise UnsupportedFeatureError( + self.name, + "partition tree introspection", + "Partition tree introspection requires PostgreSQL 10+.", + ) + if expr.include_stats: + stats_columns = """ + CASE WHEN cls.reltuples < 0 THEN NULL + ELSE cls.reltuples::bigint END AS row_count, + pg_total_relation_size(tree.oid) AS total_bytes,""" + else: + stats_columns = """ + NULL::bigint AS row_count, + NULL::bigint AS total_bytes,""" + sql = f""" + WITH RECURSIVE partition_tree AS ( + SELECT cls.oid, cls.relname AS name, 0 AS level, + NULL::oid AS parent_oid + FROM pg_class cls + WHERE cls.oid = to_regclass({self.p()}) + UNION ALL + SELECT child.oid, child.relname, tree.level + 1, tree.oid + FROM partition_tree tree + JOIN pg_inherits inh ON inh.inhparent = tree.oid + JOIN pg_class child ON child.oid = inh.inhrelid + ) + SELECT tree.name AS name, + tree.level AS level, + parent.relname AS parent, + ns.nspname AS schema_name, + pg_get_partkeydef(tree.oid) AS partition_key, + pg_get_expr(cls.relpartbound, cls.oid) AS bound,{stats_columns} + cls.relkind = 'p' AS is_partitioned + FROM partition_tree tree + JOIN pg_class cls ON cls.oid = tree.oid + JOIN pg_namespace ns ON ns.oid = cls.relnamespace + LEFT JOIN pg_class parent ON parent.oid = tree.parent_oid + ORDER BY tree.level, tree.name + """ + return sql, (expr.qualified_name,) + def format_partition_metadata_query_statement( self, expr: "PostgresPartitionMetadataQueryStatement" ) -> Tuple[str, tuple]: diff --git a/src/rhosocial/activerecord/backend/impl/postgres/partition/__init__.py b/src/rhosocial/activerecord/backend/impl/postgres/partition/__init__.py new file mode 100644 index 00000000..52c5c124 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/partition/__init__.py @@ -0,0 +1,65 @@ +# src/rhosocial/activerecord/backend/impl/postgres/partition/__init__.py +"""PostgreSQL partition plans and maintenance. + +Partition strategy lives in the backend, not on an ActiveRecord model: a +model only declares the primary-key shape a partitioned table needs. This +package describes PostgreSQL trees and maintains them: + +* :class:`PostgresPartitionPlan` describes a parent, its key, and either one + generated policy or explicit first-level branches with their own policies. +* :class:`PostgresPartitionMaintenance` and + :class:`AsyncPostgresPartitionMaintenance` reconcile that description with + the live catalog (``ensure`` / ``list`` / ``drop_before`` / ``drop_branch`` / + ``validate``). + +The module is owner-agnostic: it never hard-codes an application's table names +or retention defaults. +""" + +from .bounds import Interval, add_interval, default_label, iter_range_bounds, normalize_interval +from .maintenance import ( + AsyncPostgresPartitionMaintenance, + PostgresPartitionMaintenance, +) +from .plan import ( + HashBranch, + HashPolicy, + ListBranch, + ListPolicy, + PartitionPolicy, + PartitionSpec, + PostgresPartitionPlan, + RangeBranch, + RangePolicy, +) +from .report import ( + PairedDifference, + PartitionBatchReport, + PartitionInfo, + PartitionOperation, + PartitionReport, +) + +__all__ = [ + "AsyncPostgresPartitionMaintenance", + "HashBranch", + "HashPolicy", + "Interval", + "ListBranch", + "ListPolicy", + "PairedDifference", + "PartitionBatchReport", + "PartitionInfo", + "PartitionOperation", + "PartitionPolicy", + "PartitionReport", + "PartitionSpec", + "PostgresPartitionMaintenance", + "PostgresPartitionPlan", + "RangeBranch", + "RangePolicy", + "add_interval", + "default_label", + "iter_range_bounds", + "normalize_interval", +] diff --git a/src/rhosocial/activerecord/backend/impl/postgres/partition/bounds.py b/src/rhosocial/activerecord/backend/impl/postgres/partition/bounds.py new file mode 100644 index 00000000..d067eb00 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/partition/bounds.py @@ -0,0 +1,166 @@ +# src/rhosocial/activerecord/backend/impl/postgres/partition/bounds.py +"""Boundary arithmetic for PostgreSQL range partition policies. + +A range policy speaks in half-open intervals ``[start, end)``. This module +owns the two things every such policy needs and nothing else: + +* how a named interval advances a ``date``/``datetime`` value, and +* how the start of an interval is turned into a partition-name label. + +Both are pure functions, so the policy objects and the maintenance executor +can agree on boundaries without one of them owning a calendar. +""" + +import calendar +from datetime import date, datetime, timedelta +from typing import Any, Iterator, Optional, Tuple, Union + +__all__ = [ + "Interval", + "add_interval", + "default_label", + "iter_range_bounds", + "normalize_interval", +] + +#: A named unit or an ``(amount, unit)`` pair, e.g. ``"month"`` or ``(15, "day")``. +Interval = Union[str, Tuple[int, str]] + +_NAMED_UNITS = ("day", "month", "quarter", "year") +_UNIT_MONTHS = {"month": 1, "quarter": 3, "year": 12} + + +def normalize_interval(interval: Interval) -> Tuple[int, str]: + """Return ``(amount, unit)`` for a policy interval. + + Accepted spellings are a unit name (``"day"``, ``"month"``, ``"quarter"``, + ``"year"``), a countable spelling such as ``"15 day"`` / ``"1 month"``, + and an ``(amount, unit)`` pair. The amount must be a positive integer. + + Raises: + ValueError: if the interval is not one of these spellings. + TypeError: if the interval is not a string or a pair. + """ + if isinstance(interval, tuple): + if len(interval) != 2: + raise ValueError("interval pairs must be (amount, unit)") + amount, unit = interval + if isinstance(amount, bool) or not isinstance(amount, int) or amount <= 0: + raise ValueError("interval amount must be a positive integer") + elif isinstance(interval, str): + parts = interval.strip().lower().split() + if len(parts) == 1: + amount, unit = 1, parts[0] + elif len(parts) == 2: + try: + amount = int(parts[0]) + except ValueError as exc: + raise ValueError( + "interval amount must be a positive integer, got %r" % parts[0] + ) from exc + unit = parts[1] + else: + raise ValueError("interval must be a unit name or ' '") + if amount <= 0: + raise ValueError("interval amount must be a positive integer") + else: + raise TypeError("interval must be a string or an (amount, unit) pair") + + if unit not in _NAMED_UNITS: + raise ValueError( + "interval unit must be one of %s, got %r" % (", ".join(_NAMED_UNITS), unit) + ) + return amount, unit + + +def _shift_months(value: Any, months: int) -> Any: + """Advance a date or datetime by *months*, clamping the day of month.""" + month_index = value.month - 1 + months + year = value.year + month_index // 12 + month = month_index % 12 + 1 + day = min(value.day, calendar.monthrange(year, month)[1]) + return value.replace(year=year, month=month, day=day) + + +def add_interval(value: Any, interval: Interval) -> Any: + """Advance *value* by one step of *interval*. + + ``date`` and ``datetime`` values are supported; a ``datetime`` keeps its + time and tzinfo, which is what keeps timestamptz bounds in UTC independent + of any session setting. Calendar units clamp the day of month, so advancing + January 31 by one month yields February 28 rather than rolling into March. + """ + return _add_interval_steps(value, interval, 1) + + +def _add_interval_steps(value: Any, interval: Interval, steps: int) -> Any: + """Advance *value* by *steps* whole steps of *interval* from its origin.""" + if not isinstance(value, (date, datetime)): + raise TypeError("bound values must be date or datetime, got %r" % type(value)) + amount, unit = normalize_interval(interval) + if unit == "day": + return value + timedelta(days=amount * steps) + return _shift_months(value, _UNIT_MONTHS[unit] * amount * steps) + + +def default_label(start: Any, interval: Interval) -> str: + """Derive the default partition-name label from an interval start. + + Monthly plans read ``2026_01``, quarterly ones ``2026_q1``, yearly ones + ``2026``, and day-granular plans (including ``(15, "day")``) use + ``2026_01_01``. The label is the suffix of a generated partition name, so + it deliberately contains no separator beyond ``_``. + """ + amount, unit = normalize_interval(interval) + if unit == "year": + return start.strftime("%Y") + if unit == "quarter": + return "%s_q%d" % (start.strftime("%Y"), (start.month - 1) // 3 + 1) + if unit == "month": + return start.strftime("%Y_%m") + return start.strftime("%Y_%m_%d") + + +def iter_range_bounds( + start: Any, + interval: Interval = "month", + count: Optional[int] = None, + through: Optional[Any] = None, +) -> Iterator[Tuple[Any, Any]]: + """Yield ``(start, end)`` half-open ranges from *start*. + + Generation stops at the first of two limits: *count* boundaries have been + produced, or the next start would not be before *through*. Both limits may + be supplied; at least one is required. + + Args: + start: First boundary, as ``date`` or ``datetime``. + interval: Step between boundaries. + count: Maximum number of ranges to yield. + through: Exclusive upper limit. A range whose start is at or beyond + this value is not produced. + + Raises: + TypeError: if *start* is not a date or datetime. + ValueError: if neither limit is given, or *count* is not positive. + """ + if not isinstance(start, (date, datetime)): + raise TypeError("start must be date or datetime, got %r" % type(start)) + if count is None and through is None: + raise ValueError("iter_range_bounds requires count or through") + if count is not None: + if isinstance(count, bool) or not isinstance(count, int) or count <= 0: + raise ValueError("count must be a positive integer") + if through is not None and through <= start: + return + + produced = 0 + while True: + if count is not None and produced >= count: + return + current = _add_interval_steps(start, interval, produced) + if through is not None and current >= through: + return + end = _add_interval_steps(start, interval, produced + 1) + yield current, end + produced += 1 diff --git a/src/rhosocial/activerecord/backend/impl/postgres/partition/maintenance.py b/src/rhosocial/activerecord/backend/impl/postgres/partition/maintenance.py new file mode 100644 index 00000000..9ea98194 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/partition/maintenance.py @@ -0,0 +1,1627 @@ +# src/rhosocial/activerecord/backend/impl/postgres/partition/maintenance.py +"""PostgreSQL partition maintenance: plan in, tree out, DDL in units. + +The executor is deliberately owner-agnostic. It knows a plan's parent, keys, +policies and name pattern; it knows nothing about which application owns the +table, which retention decision produced the cutoff, or which application +table a partition belongs to. + +PostgreSQL's DDL is transactional, and the executor uses that: each create/drop +*unit* runs inside one transaction. A unit is one branch together with its +leaves, or all leaves of a flat (single-level) plan. A failed unit leaves no +partially-built branch behind; sibling branches are independent, as the plan +asks for heterogeneous branches. + +Public surface is mirrored sync/async: + +* :class:`PostgresPartitionMaintenance` +* :class:`AsyncPostgresPartitionMaintenance` + +Both expose ``ensure``, ``list``, ``drop_before``, ``drop_branch``, +``validate`` and the batch forms ``ensure_many``, ``list_many`` and +``drop_before_many``. +""" + +import re +from dataclasses import replace +from datetime import date, datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from rhosocial.activerecord.backend.expression.core import Column +from rhosocial.activerecord.backend.expression.objects import Table +from rhosocial.activerecord.backend.expression.statements import ( + DropTableStatement, + PartitionClause, +) +from rhosocial.activerecord.backend.impl.postgres.expression.ddl.partition import ( + PostgresCreatePartitionStatement, + PostgresDetachPartitionStatement, + PostgresPartitionTreeQueryStatement, +) +from .bounds import normalize_interval +from .plan import ( + ListPolicy, + PartitionBranch, + PartitionSpec, + PostgresPartitionPlan, + RangePolicy, +) +from .report import ( + PairedDifference, + PartitionBatchReport, + PartitionInfo, + PartitionOperation, + PartitionReport, +) + +__all__ = [ + "AsyncPostgresPartitionMaintenance", + "PostgresPartitionMaintenance", +] + +_METHOD_WORDS = ("RANGE", "LIST", "HASH") + +_RANGE_BOUND_RE = re.compile( + r"^FOR VALUES FROM \((?P.*)\) TO \((?P.*)\)$", re.IGNORECASE +) +_LIST_BOUND_RE = re.compile(r"^FOR VALUES IN \((?P.*)\)$", re.IGNORECASE) +_HASH_BOUND_RE = re.compile( + r"^FOR VALUES WITH \(\s*MODULUS\s+(?P\d+)\s*,\s*" + r"REMAINDER\s+(?P\d+)\s*\)$", + re.IGNORECASE, +) +_TEMPORAL_RE = re.compile( + r"^(?P\d{4})-(?P\d{2})-(?P\d{2})" + r"(?:[ T](?P\d{2}):(?P\d{2})" + r"(?::(?P\d{2})(?:\.(?P\d{1,6}))?)?" + r"(?PZ|[+-]\d{2}(?::?\d{2})?)?)?$", + re.IGNORECASE, +) +_MIDNIGHT = datetime.min.time() +_UNPARSEABLE_WORDS = ("MINVALUE", "MAXVALUE", "NULL") + + +def _unquote_bound_value(text: str) -> str: + """Strip a single-quoted PostgreSQL literal and undo doubled quotes.""" + text = text.strip() + if len(text) >= 2 and text[0] == "'" and text[-1] == "'": + return text[1:-1].replace("''", "'") + return text + + +def _split_bound_values(text: str) -> Tuple[str, ...]: + """Split a LIST bound's comma-separated values, respecting quotes.""" + tokens: List[str] = [] + start = 0 + quoted = False + index = 0 + while index < len(text): + character = text[index] + if character == "'": + if quoted and index + 1 < len(text) and text[index + 1] == "'": + index += 2 + continue + quoted = not quoted + elif character == "," and not quoted: + tokens.append(text[start:index].strip()) + start = index + 1 + index += 1 + tail = text[start:].strip() + if tail: + tokens.append(tail) + return tuple(tokens) + + +def _parse_list_value(token: str) -> Any: + """Parse one LIST bound token into a Python scalar when possible.""" + token = token.strip() + if len(token) >= 2 and token[0] == "'" and token[-1] == "'": + return token[1:-1].replace("''", "'") + if token.upper() == "NULL": + return None + try: + return int(token) + except ValueError: + pass + try: + return float(token) + except ValueError: + return token + + +def _parse_temporal(text: str) -> Optional[Any]: + """Parse PostgreSQL's date/timestamp bound text into a Python value.""" + text = _unquote_bound_value(text) + if text.upper() in _UNPARSEABLE_WORDS: + return None + match = _TEMPORAL_RE.match(text) + if match is None: + return None + parts = match.groupdict() + try: + year = int(parts["year"]) + month = int(parts["month"]) + day = int(parts["day"]) + except (TypeError, ValueError): + return None + if parts["hour"] is None: + try: + return date(year, month, day) + except ValueError: + return None + microsecond = 0 + if parts["fraction"]: + microsecond = int((parts["fraction"] + "000000")[:6]) + tzinfo = None + offset = parts["offset"] + if offset: + if offset.upper() == "Z": + tzinfo = timezone.utc + else: + sign = -1 if offset[0] == "-" else 1 + hours = int(offset[1:3]) + minutes = int(offset[4:6]) if len(offset) > 3 else 0 + tzinfo = timezone(sign * timedelta(hours=hours, minutes=minutes)) + try: + return datetime( + year, month, day, + int(parts["hour"]), int(parts["minute"]), + int(parts["second"] or 0), microsecond, tzinfo, + ) + except ValueError: + return None + + +def _parse_partition_bound(text: Optional[str]) -> Optional[Dict[str, Any]]: + """Parse ``relpartbound`` text; return ``None`` when it is unparseable. + + Supported PostgreSQL shapes are RANGE ``FROM ... TO ...`` (date and + timestamp values), LIST ``IN (...)`` and HASH ``MODULUS ... REMAINDER``. + """ + if not isinstance(text, str): + return None + text = text.strip() + if not text: + return None + match = _RANGE_BOUND_RE.match(text) + if match is not None: + parsed: Dict[str, Any] = {"kind": "RANGE"} + start = _parse_temporal(match.group("start")) + end = _parse_temporal(match.group("end")) + if start is not None: + parsed["range_start"] = start + if end is not None: + parsed["range_end"] = end + return parsed + match = _LIST_BOUND_RE.match(text) + if match is not None: + return { + "kind": "LIST", + "list_values": tuple( + _parse_list_value(token) + for token in _split_bound_values(match.group("values")) + ), + } + match = _HASH_BOUND_RE.match(text) + if match is not None: + return { + "kind": "HASH", + "modulus": int(match.group("modulus")), + "remainder": int(match.group("remainder")), + } + return None + + +def _same_bound(actual: Any, expected: Any) -> bool: + """Compare parsed catalog bounds with plan bounds. + + The server may render a date where the plan holds a midnight datetime, or + render a timestamp without an offset where the plan holds an aware one, so + the comparison normalises those representational differences. + """ + if isinstance(actual, datetime) and isinstance(expected, datetime): + if actual.tzinfo is not None and expected.tzinfo is not None: + return actual.astimezone(timezone.utc) == expected.astimezone(timezone.utc) + if actual.tzinfo is not None or expected.tzinfo is not None: + return actual.replace(tzinfo=None) == expected.replace(tzinfo=None) + return actual == expected + actual_is_datetime = isinstance(actual, datetime) + expected_is_datetime = isinstance(expected, datetime) + if actual_is_datetime != expected_is_datetime: + moment = actual if actual_is_datetime else expected + day = expected if actual_is_datetime else actual + return moment.date() == day and moment.time() == _MIDNIGHT + return actual == expected + + +def _format_bound_value(value: Any) -> str: + if isinstance(value, datetime): + return value.isoformat(sep=" ") + return str(value) + + +def _normalise_bound_text(text: Optional[str]) -> Optional[str]: + """Collapse whitespace in a catalog bound; ``None`` when it is empty.""" + if not isinstance(text, str): + return None + normalised = " ".join(text.split()) + return normalised or None + + +def _key_value(value: Any) -> str: + """Render one bucket component in a stable, comparable form. + + A midnight timestamp and a date are the same boundary in the plan's + half-open grammar, so they share one key even when the catalog spells them + differently. The raw text comparison still reports that spelling drift. + """ + if isinstance(value, datetime): + if value.time() == _MIDNIGHT: + return value.date().isoformat() + return value.isoformat(sep=" ") + if isinstance(value, date): + return value.isoformat() + if isinstance(value, str): + return repr(value) + return str(value) + + +def _sorted_values(values: Sequence[Any]) -> Tuple[Any, ...]: + """Sort LIST values, falling back to type and repr for mixed types.""" + try: + return tuple(sorted(values)) + except TypeError: + return tuple( + sorted(values, key=lambda value: (type(value).__name__, repr(value))) + ) + + +def _extras_bucket(info: PartitionInfo) -> Optional[str]: + """Return the bucket key the row's parsed extras describe, if any.""" + method = (info.method or "").upper() + extras = info.extras or {} + if method == "RANGE": + if "range_start" in extras and "range_end" in extras: + return "RANGE|%s|%s" % ( + _key_value(extras["range_start"]), + _key_value(extras["range_end"]), + ) + elif method == "HASH": + if "modulus" in extras and "remainder" in extras: + return "HASH|%s|%s" % ( + _key_value(extras["modulus"]), + _key_value(extras["remainder"]), + ) + elif method == "LIST": + if "list_values" in extras: + values = _sorted_values(extras["list_values"]) + return "LIST|%s" % ",".join(_key_value(value) for value in values) + return None + + +def _difference_key(info: PartitionInfo) -> str: + """Return the normalized bucket a first-level row is compared on. + + Parsed ``extras`` come first because they give the same key for two + catalogues that spell the same boundary differently. ``bound`` text is the + fallback for a server whose bound parser did not recognise the shape, and + the name suffix is the last resort for a row that carries no bound at all. + """ + extras_key = _extras_bucket(info) + if extras_key is not None: + return extras_key + bound = _normalise_bound_text(info.bound) + if bound is not None: + return bound + parent = info.parent + if parent is not None and info.name.startswith(parent + "_"): + return info.name[len(parent) + 1:] + return info.name + + +def _boundary_signature(info: PartitionInfo) -> Optional[str]: + """Return the semantic boundary of a row, if it has one.""" + extras_key = _extras_bucket(info) + if extras_key is not None: + return extras_key + return _normalise_bound_text(info.bound) + + +def _boundaries_disagree(left: PartitionInfo, right: PartitionInfo) -> bool: + """Whether two rows sharing a bucket key describe different boundaries. + + Boundary text is compared verbatim (after whitespace normalization) because + two catalogues that spell the same bucket differently are exactly the drift + the paired view must surface. When either side carries no text, the parsed + extras are the best remaining evidence. + """ + left_text = _normalise_bound_text(left.bound) + right_text = _normalise_bound_text(right.bound) + if left_text is not None and right_text is not None: + return left_text != right_text + return _boundary_signature(left) != _boundary_signature(right) + + +def _policy_shape(policy: Any) -> Optional[Tuple[Any, ...]]: + """Describe a policy's structure, ignoring the boundaries it generates.""" + if policy is None: + return None + if isinstance(policy, RangePolicy): + explicit = policy.bounds is not None + interval = None if explicit else normalize_interval(policy.interval) + return ("RANGE", tuple(policy.columns), interval, explicit) + if isinstance(policy, ListPolicy): + return ("LIST", tuple(policy.columns)) + return ("HASH", tuple(policy.columns), policy.modulus) + + +def _branch_shape(branch: PartitionBranch) -> Tuple[Any, ...]: + """Describe a first-level branch without its name or boundary values.""" + return (type(branch).__name__, _policy_shape(branch.child)) + + +def _plan_shape(plan: PostgresPartitionPlan) -> Tuple[Any, ...]: + """Describe the structure two paired plans must share to be comparable.""" + return ( + plan.method, + tuple(plan.columns), + plan.modulus, + _policy_shape(plan.policy), + tuple(_branch_shape(branch) for branch in plan.branches), + ) + + +class _PartitionMaintenanceBase: + """Plan/catalog/DDL logic shared by the sync and async executors. + + Subclasses implement only the three I/O seams: ``_fetch_rows``, ``_execute`` + and ``_run_units``. Everything else -- expected specs, missing/orphan + comparison, continuity checks, DDL rendering and unit grouping -- lives + here once, so the two executors cannot drift apart. + """ + + def __init__(self, backend: Any) -> None: + self.backend = backend + self.dialect = backend.dialect + + # ------------------------------------------------------------------ I/O + def _fetch_rows(self, sql: str, params: Tuple[Any, ...]): + raise NotImplementedError + + def _execute(self, sql: str, params: Tuple[Any, ...]) -> None: + raise NotImplementedError + + def _run_units(self, plan: PostgresPartitionPlan, units, callback) -> None: + raise NotImplementedError + + # --------------------------------------------------------------- catalog + def _tree_expression(self, plan: PostgresPartitionPlan): + return PostgresPartitionTreeQueryStatement( + self.dialect, + root_table=plan.parent, + schema=plan.schema, + include_stats=True, + ) + + def _infos(self, plan: PostgresPartitionPlan) -> Tuple[PartitionInfo, ...]: + sql, params = self._tree_expression(plan).to_sql() + rows = self._fetch_rows(sql, params) + return self._parse_rows(rows) + + @staticmethod + def _parse_rows(rows: Sequence[Any]) -> Tuple[PartitionInfo, ...]: + infos: List[PartitionInfo] = [] + for row in rows: + raw_key = row.get("partition_key") + method = None + if raw_key: + head = raw_key.split("(")[0].strip().upper() + if head in _METHOD_WORDS: + method = head + row_count = row.get("row_count") + total_bytes = row.get("total_bytes") + bound = row.get("bound") + parsed = _parse_partition_bound(bound) + extras: Dict[str, Any] = {} + if parsed is not None: + extras = { + key: value + for key, value in parsed.items() + if key != "kind" + } + infos.append( + PartitionInfo( + name=row["name"], + parent=row.get("parent"), + level=int(row.get("level", 0) or 0), + schema=row.get("schema_name"), + method=method, + bound=bound, + row_count=None if row_count is None else int(row_count), + size_bytes=None if total_bytes is None else int(total_bytes), + is_partitioned=bool(row.get("is_partitioned")), + extras=extras, + ) + ) + return tuple(infos) + + # ---------------------------------------------------------------- expected + def _expected( + self, plan: PostgresPartitionPlan, through: Optional[Any] + ) -> Tuple[PartitionSpec, ...]: + specs = tuple(plan.iter_specs(through=through)) + if not specs: + raise ValueError("plan describes no partitions") + return specs + + def _safe_expected( + self, plan: PostgresPartitionPlan, through: Optional[Any] = None + ) -> Tuple[PartitionSpec, ...]: + """Return expected specs, or nothing when the plan has no horizon.""" + try: + return self._expected(plan, through) + except ValueError: + if plan.is_fully_bounded(through=through): + raise + return () + + @staticmethod + def _actual_names(infos: Sequence[PartitionInfo]) -> Dict[str, PartitionInfo]: + return {info.name: info for info in infos if info.level > 0} + + def _boundary_issue( + self, spec: PartitionSpec, info: PartitionInfo + ) -> Optional[str]: + """Describe a parsed-bound mismatch or an unparseable bound.""" + parsed = _parse_partition_bound(info.bound) + if parsed is None: + return "unparsed bound for %s: %s" % ( + spec.name, info.bound if info.bound else "" + ) + kind = parsed.get("kind") + if kind != spec.method: + return "boundary mismatch for %s: catalog %s bound, plan %s bound" % ( + spec.name, kind, spec.method, + ) + if spec.method == "RANGE": + start = parsed.get("range_start") + end = parsed.get("range_end") + if start is None or end is None: + return "unparsed bound for %s: %s" % (spec.name, info.bound) + if _same_bound(start, spec.range_start) and _same_bound( + end, spec.range_end + ): + return None + return ( + "boundary mismatch for %s: catalog %s..%s, plan %s..%s" + % ( + spec.name, + _format_bound_value(start), + _format_bound_value(end), + _format_bound_value(spec.range_start), + _format_bound_value(spec.range_end), + ) + ) + if spec.method == "LIST": + plan_values = tuple(spec.bounds.get("values", ())) + actual_values = tuple(parsed.get("list_values", ())) + if set(actual_values) == set(plan_values): + return None + return "boundary mismatch for %s: catalog values %s, plan values %s" % ( + spec.name, actual_values, plan_values, + ) + plan_modulus = spec.bounds.get("modulus") + plan_remainder = spec.bounds.get("remainder") + if ( + parsed.get("modulus") == plan_modulus + and parsed.get("remainder") == plan_remainder + ): + return None + return ( + "boundary mismatch for %s: catalog modulus %s remainder %s, " + "plan modulus %s remainder %s" + % ( + spec.name, + parsed.get("modulus"), + parsed.get("remainder"), + plan_modulus, + plan_remainder, + ) + ) + + def _boundary_mismatch( + self, spec: PartitionSpec, info: PartitionInfo + ) -> bool: + """Whether a parseable catalog bound disagrees with the plan's spec.""" + parsed = _parse_partition_bound(info.bound) + if parsed is None: + return False + if parsed.get("kind") != spec.method: + return True + if spec.method == "RANGE": + start = parsed.get("range_start") + end = parsed.get("range_end") + if start is None or end is None: + return False + return not ( + _same_bound(start, spec.range_start) + and _same_bound(end, spec.range_end) + ) + if spec.method == "LIST": + plan_values = tuple(spec.bounds.get("values", ())) + return set(parsed.get("list_values", ())) != set(plan_values) + return ( + parsed.get("modulus") != spec.bounds.get("modulus") + or parsed.get("remainder") != spec.bounds.get("remainder") + ) + + def _needs_partition( + self, spec: PartitionSpec, actual: Dict[str, PartitionInfo] + ) -> bool: + info = actual.get(spec.name) + if info is None: + return True + return self._boundary_mismatch(spec, info) + + def _missing( + self, + specs: Sequence[PartitionSpec], + actual: Dict[str, PartitionInfo], + ) -> Tuple[Tuple[str, str], ...]: + return tuple( + (spec.parent, spec.name) + for spec in specs + if self._needs_partition(spec, actual) + ) + + def _orphans( + self, + plan: PostgresPartitionPlan, + specs: Sequence[PartitionSpec], + actual: Dict[str, PartitionInfo], + through: Optional[Any], + ) -> Tuple[str, ...]: + if not plan.is_fully_bounded(through=through): + return () + expected_names = {spec.name for spec in specs} + return tuple( + sorted(name for name in actual if name not in expected_names) + ) + + # -------------------------------------------------------------- rendering + def _create_statement( + self, + plan: PostgresPartitionPlan, + spec: PartitionSpec, + if_not_exists: bool, + ) -> Tuple[str, Tuple[Any, ...]]: + clause = None + if spec.child_method is not None: + clause = PartitionClause( + self.dialect, + spec.child_method, + [Column(self.dialect, name) for name in spec.child_columns], + ) + statement = PostgresCreatePartitionStatement( + dialect=self.dialect, + partition_name=spec.name, + parent_table=spec.parent, + partition_type=spec.method, + partition_values=dict(spec.bounds), + schema=plan.schema, + parent_schema=plan.schema, + if_not_exists=if_not_exists, + partition_clause=clause, + ) + return statement.to_sql() + + def _drop_statement( + self, + plan: PostgresPartitionPlan, + name: str, + cascade: bool, + ) -> Tuple[str, Tuple[Any, ...]]: + statement = DropTableStatement( + dialect=self.dialect, + table=Table(self.dialect, name, schema_name=plan.schema), + if_exists=True, + cascade=cascade, + ) + return statement.to_sql() + + def _detach_statement( + self, plan: PostgresPartitionPlan, spec: PartitionSpec + ) -> Tuple[str, Tuple[Any, ...]]: + statement = PostgresDetachPartitionStatement( + dialect=self.dialect, + partition_name=spec.name, + parent_table=spec.parent, + schema=plan.schema, + parent_schema=plan.schema, + concurrently=True, + ) + return statement.to_sql() + + def _drop_statements( + self, + plan: PostgresPartitionPlan, + spec: PartitionSpec, + concurrent_detach: bool, + ) -> Tuple[Tuple[str, str, Tuple[Any, ...]], ...]: + """Return the ``(action, sql, params)`` sequence for one dropped leaf. + + ``DETACH PARTITION CONCURRENTLY`` is only rendered when the caller asks + for it *and* the server supports it; every other case keeps the single + transactional ``DROP TABLE`` path. + """ + concurrent = ( + concurrent_detach and self.dialect.supports_concurrent_detach() + ) + drop_sql, drop_params = self._drop_statement(plan, spec.name, cascade=False) + if not concurrent: + return (("drop", drop_sql, drop_params),) + detach_sql, detach_params = self._detach_statement(plan, spec) + return ( + ("detach", detach_sql, detach_params), + ("drop", drop_sql, drop_params), + ) + + @staticmethod + def _units( + plan: PostgresPartitionPlan, + specs: Sequence[PartitionSpec], + ) -> Tuple[Tuple[PartitionSpec, ...], ...]: + """Group specs into transactional units. + + A flat plan (one policy directly below the parent) is a single unit: its + leaves are created or dropped together. A multi-level plan is one unit + per first-level branch, in declaration order, with the branch ahead of + its own leaves. + """ + if plan.policy is not None: + return (tuple(specs),) if specs else () + grouped: Dict[str, List[PartitionSpec]] = {} + order: List[str] = [] + for spec in specs: + key = spec.name if spec.level == 1 else spec.parent + if key not in grouped: + grouped[key] = [] + order.append(key) + grouped[key].append(spec) + return tuple(tuple(grouped[key]) for key in order) + + # -------------------------------------------------------------- validation + def _continuity_issues( + self, specs: Sequence[PartitionSpec] + ) -> Tuple[str, ...]: + issues: List[str] = [] + by_parent: Dict[str, List[PartitionSpec]] = {} + for spec in specs: + if spec.range_start is None or spec.range_end is None: + continue + by_parent.setdefault(spec.parent, []).append(spec) + for parent, group in sorted(by_parent.items()): + ordered = sorted(group, key=lambda item: item.range_start) + for previous, current in zip(ordered, ordered[1:]): + if current.range_start > previous.range_end: + issues.append( + "gap in %s between %s and %s" + % (parent, previous.name, current.name) + ) + elif current.range_start < previous.range_end: + issues.append( + "overlap in %s between %s and %s" + % (parent, previous.name, current.name) + ) + return tuple(issues) + + def _structure_issues( + self, + plan: PostgresPartitionPlan, + infos: Sequence[PartitionInfo], + specs: Sequence[PartitionSpec], + ) -> Tuple[str, ...]: + issues: List[str] = [] + actual = self._actual_names(infos) + roots = [info for info in infos if info.level == 0] + if not roots: + issues.append("partitioned parent %s does not exist" % plan.parent) + return tuple(issues) + root = roots[0] + if root.method != plan.method: + issues.append( + "parent %s is partitioned by %s, plan says %s" + % (plan.parent, root.method, plan.method) + ) + for spec in specs: + info = actual.get(spec.name) + if info is None: + continue + if info.parent != spec.parent: + issues.append( + "%s belongs to %s, plan expects %s" + % (spec.name, info.parent, spec.parent) + ) + if spec.child_method is not None: + if not info.is_partitioned: + issues.append( + "branch %s is not further partitioned; %s leaves are missing" + % (spec.name, spec.child_method) + ) + elif info.method != spec.child_method: + issues.append( + "branch %s is partitioned by %s, plan says %s" + % (spec.name, info.method, spec.child_method) + ) + elif info.is_partitioned: + issues.append( + "leaf %s is partitioned; plan expects it to be a leaf" % spec.name + ) + bound_issue = self._boundary_issue(spec, info) + if bound_issue is not None: + issues.append(bound_issue) + return tuple(issues) + + def _report( + self, + plan: PostgresPartitionPlan, + infos: Sequence[PartitionInfo], + specs: Sequence[PartitionSpec], + through: Optional[Any], + operations: Sequence[PartitionOperation] = (), + dry_run: bool = False, + ) -> PartitionReport: + actual = self._actual_names(infos) + issues = self._structure_issues(plan, infos, specs) + self._continuity_issues(specs) + return PartitionReport( + target=plan.parent, + partitions=tuple(infos), + missing=self._missing(specs, actual), + orphans=self._orphans(plan, specs, actual, through), + issues=issues, + operations=tuple(operations), + dry_run=dry_run, + ) + + def _render_creates( + self, + plan: PostgresPartitionPlan, + specs: Sequence[PartitionSpec], + if_not_exists: bool, + ) -> Tuple[PartitionOperation, ...]: + return tuple( + PartitionOperation( + action="create", + name=spec.name, + parent=spec.parent, + sql=self._create_statement(plan, spec, if_not_exists)[0], + level=spec.level, + ) + for spec in specs + ) + + @staticmethod + def _without( + specs: Sequence[PartitionSpec], names: Sequence[str] + ) -> Tuple[PartitionSpec, ...]: + removed = set(names) + return tuple(spec for spec in specs if spec.name not in removed) + + def _drop_candidates( + self, + plan: PostgresPartitionPlan, + cutoff: Any, + infos: Sequence[PartitionInfo], + through: Optional[Any], + ) -> Tuple[PartitionSpec, ...]: + if not isinstance(cutoff, (date, datetime)): + raise TypeError("cutoff must be a date or datetime") + actual = self._actual_names(infos) + specs = self._expected(plan, through=through if through is not None else cutoff) + candidates: List[PartitionSpec] = [] + for spec in specs: + if not spec.is_leaf or spec.range_end is None: + continue + try: + early = spec.range_end <= cutoff + except TypeError as exc: + raise TypeError( + "cutoff must match the plan's bound type (date vs datetime)" + ) from exc + info = actual.get(spec.name) + if early and info is not None and not info.is_partitioned: + candidates.append(spec) + return tuple(candidates) + + # ------------------------------------------------------------------ batch + @staticmethod + def _resolve_batch(plans, pairs): + """Normalize both batch spellings and de-duplicate their targets.""" + pair_entries: List[ + Tuple[str, PostgresPartitionPlan, PostgresPartitionPlan] + ] = [] + if pairs is not None: + for entry in pairs: + try: + name, left, right = entry + except (TypeError, ValueError) as exc: + raise ValueError( + "a pair must be (name, left_plan, right_plan)" + ) from exc + if not isinstance(name, str) or not name.strip(): + raise ValueError("pair name must be a non-empty string") + pair_entries.append((name, left, right)) + batch: List[PostgresPartitionPlan] = [] + if plans is not None: + if isinstance(plans, PostgresPartitionPlan): + batch.append(plans) + else: + batch.extend(plans) + for _, left, right in pair_entries: + batch.extend((left, right)) + if not batch: + raise ValueError("batch requires at least one plan") + unique: List[PostgresPartitionPlan] = [] + seen: Dict[Tuple[Optional[str], str], bool] = {} + for plan in batch: + if not isinstance(plan, PostgresPartitionPlan): + raise TypeError("batch plans must be PostgresPartitionPlan values") + key = (plan.schema, plan.parent) + if key in seen: + continue + seen[key] = True + unique.append(plan) + return tuple(pair_entries), tuple(unique) + + def _ensure_batch_work( + self, + plan: PostgresPartitionPlan, + infos: Sequence[PartitionInfo], + through: Optional[Any], + if_not_exists: bool, + ): + """Return specs, rendered operations and create statements for a plan.""" + if not plan.is_fully_bounded(through=through): + raise ValueError("plan has no finite horizon") + if not infos: + raise ValueError("partitioned parent %s does not exist" % plan.parent) + specs = self._expected(plan, through) + actual = self._actual_names(infos) + operations: List[PartitionOperation] = [] + statements: List[Tuple[str, Tuple[Any, ...]]] = [] + for spec in specs: + if not self._needs_partition(spec, actual): + continue + sql, params = self._create_statement(plan, spec, if_not_exists) + statements.append((sql, params)) + operations.append( + PartitionOperation("create", spec.name, spec.parent, sql, spec.level) + ) + return specs, tuple(operations), tuple(statements) + + def _drop_batch_work( + self, + plan: PostgresPartitionPlan, + infos: Sequence[PartitionInfo], + cutoff: Any, + through: Optional[Any], + concurrent_detach: bool, + ): + """Return specs, candidates, rendered operations and drop statements.""" + specs = self._expected(plan, through=through if through is not None else cutoff) + candidates = self._drop_candidates(plan, cutoff, infos, through) + operations: List[PartitionOperation] = [] + statements: List[Tuple[str, Tuple[Any, ...]]] = [] + for spec in candidates: + for action, sql, params in self._drop_statements( + plan, spec, concurrent_detach + ): + operations.append( + PartitionOperation(action, spec.name, spec.parent, sql, spec.level) + ) + statements.append((sql, params)) + return specs, candidates, tuple(operations), tuple(statements) + + @staticmethod + def _bucket_map( + pair_name: str, + side: str, + partitions: Sequence[PartitionInfo], + issues: List[str], + ) -> Dict[str, PartitionInfo]: + """Map level-1 rows to their bucket key for one side of a pair.""" + buckets: Dict[str, PartitionInfo] = {} + for info in partitions: + if info.level != 1: + continue + key = _difference_key(info) + if key in buckets: + issues.append( + "pair %s has duplicate bucket %s in %s" + % (pair_name, key, side) + ) + continue + buckets[key] = info + return buckets + + def _paired_report( + self, + pair_entries, + reports: Dict[str, PartitionReport], + ) -> Tuple[Tuple[PairedDifference, ...], Tuple[str, ...]]: + """Compare the declarations a caller made and describe only real drift.""" + differences: List[PairedDifference] = [] + issues: List[str] = [] + for name, left, right in pair_entries: + if _plan_shape(left) != _plan_shape(right): + issues.append("pair %s plans are not comparable" % name) + continue + left_report = reports.get(left.parent) + right_report = reports.get(right.parent) + if left_report is None or right_report is None: + issues.append("pair %s has no report for one side" % name) + continue + left_buckets = self._bucket_map(name, "left", left_report.partitions, issues) + right_buckets = self._bucket_map( + name, "right", right_report.partitions, issues + ) + for key in sorted(set(left_buckets) | set(right_buckets)): + left_info = left_buckets.get(key) + right_info = right_buckets.get(key) + if left_info is None: + assert right_info is not None + differences.append( + PairedDifference(name, key, "right", None, right_info.name) + ) + elif right_info is None: + differences.append( + PairedDifference(name, key, "left", left_info.name, None) + ) + elif _boundaries_disagree(left_info, right_info): + issues.append("boundary mismatch for %s/%s" % (name, key)) + return tuple(differences), tuple(issues) + + @staticmethod + def _batch_report( + pair_entries, + reports: Dict[str, PartitionReport], + differences: Tuple[PairedDifference, ...], + issues: Tuple[str, ...], + ) -> PartitionBatchReport: + """Assemble the per-target reports and the paired view.""" + return PartitionBatchReport( + reports=reports, + pairs=tuple(name for name, _, _ in pair_entries), + differences=differences, + issues=issues, + ) + + +class PostgresPartitionMaintenance(_PartitionMaintenanceBase): + """Synchronous PostgreSQL partition maintenance executor. + + Example: + >>> maintenance = PostgresPartitionMaintenance(backend) + >>> report = maintenance.ensure(plan, through=date(2027, 1, 1)) + >>> report.missing + () + """ + + def _fetch_rows(self, sql: str, params: Tuple[Any, ...]): + return self.backend.fetch_all(sql, params) + + def _execute(self, sql: str, params: Tuple[Any, ...]) -> None: + self.backend.execute(sql, params) + + def _run_units(self, plan, units, callback) -> None: + for unit in units: + with self.backend.transaction(): + for spec in unit: + callback(spec) + + def list( + self, + plan: PostgresPartitionPlan, + through: Optional[Any] = None, + ) -> PartitionReport: + """Read the live tree and compare it with the plan. + + Args: + plan: Tree description. + through: Exclusive horizon used to expand generated RANGE policies. + + Returns: + A report with the catalog tree, missing leaves, orphans and + continuity issues. No DDL is executed. + """ + infos = self._infos(plan) + specs = self._safe_expected(plan, through) + report = self._report(plan, infos, specs, through) + if not plan.is_fully_bounded(through=through): + report = replace( + report, issues=report.issues + ("plan has no finite horizon",) + ) + return report + + def list_many( + self, + plans: Optional[Any] = None, + *, + pairs: Optional[Any] = None, + ) -> PartitionBatchReport: + """Read several plans and report paired drift without changing anything. + + Args: + plans: Independent plans to read. + pairs: ``(pair_name, left_plan, right_plan)`` triples to compare. + + Returns: + A batch report keyed by target, plus the paired drift. + """ + pair_entries, batch_plans = self._resolve_batch(plans, pairs) + reports: Dict[str, PartitionReport] = {} + for plan in batch_plans: + reports[plan.parent] = self.list(plan) + differences, issues = self._paired_report(pair_entries, reports) + return self._batch_report(pair_entries, reports, differences, issues) + + def ensure( + self, + plan: PostgresPartitionPlan, + through: Optional[Any] = None, + if_not_exists: bool = True, + ) -> PartitionReport: + """Create every partition the plan expects but the catalog lacks. + + Existing partitions are left untouched unless their catalog boundary + disagrees with the plan; a mismatched partition is not treated as + present. Each transactional unit (one branch with its leaves, or all + leaves of a flat plan) is committed independently. + + Args: + plan: Tree description. + through: Exclusive horizon used to expand generated RANGE policies. + if_not_exists: Add ``IF NOT EXISTS`` to every created statement. + + Returns: + The post-operation report, with the executed DDL in ``operations``. + """ + if not plan.is_fully_bounded(through=through): + raise ValueError("plan has no finite horizon") + infos = self._infos(plan) + if not infos: + raise ValueError("partitioned parent %s does not exist" % plan.parent) + specs = self._expected(plan, through) + actual = self._actual_names(infos) + missing = tuple( + spec for spec in specs if self._needs_partition(spec, actual) + ) + operations: List[PartitionOperation] = [] + + def create(spec: PartitionSpec) -> None: + sql, params = self._create_statement(plan, spec, if_not_exists) + self._execute(sql, params) + operations.append( + PartitionOperation("create", spec.name, spec.parent, sql, spec.level) + ) + + self._run_units(plan, self._units(plan, missing), create) + report = self.list(plan, through) + return replace(report, operations=tuple(operations), dry_run=False) + + def ensure_many( + self, + plans: Optional[Any] = None, + *, + pairs: Optional[Any] = None, + through: Optional[Any] = None, + if_not_exists: bool = True, + ) -> PartitionBatchReport: + """Create every missing partition for several plans in one transaction. + + The whole batch is one transactional unit: a failure in any plan aborts + the batch instead of leaving earlier targets created. After a successful + run every target is re-read, so each report reflects the post-state. + + Args: + plans: Independent plans to reconcile. + pairs: ``(pair_name, left_plan, right_plan)`` triples. Both sides + are reconciled and later compared for bucket drift. + through: Exclusive horizon applied to every plan. + if_not_exists: Add ``IF NOT EXISTS`` to every created statement. + + Returns: + A batch report keyed by target, plus the paired drift. + + Raises: + ValueError: when no plan is supplied, a plan has no finite horizon, + or a plan's parent table does not exist. + """ + pair_entries, batch_plans = self._resolve_batch(plans, pairs) + prepared = [] + for plan in batch_plans: + infos = self._infos(plan) + prepared.append( + (plan,) + self._ensure_batch_work( + plan, infos, through, if_not_exists + ) + ) + with self.backend.transaction(): + for _, _, _, statements in prepared: + for sql, params in statements: + self._execute(sql, params) + reports: Dict[str, PartitionReport] = {} + for plan, specs, operations, _ in prepared: + infos = self._infos(plan) + reports[plan.parent] = self._report( + plan, infos, specs, through, operations, dry_run=False + ) + differences, issues = self._paired_report(pair_entries, reports) + return self._batch_report(pair_entries, reports, differences, issues) + + def drop_before( + self, + plan: PostgresPartitionPlan, + cutoff: Any, + dry_run: bool = True, + through: Optional[Any] = None, + concurrent_detach: bool = False, + ) -> PartitionReport: + """Drop every leaf whose whole range ends at or before *cutoff*. + + Only true leaves are considered: a branch that is itself partitioned is + never dropped here, so ``drop_before`` can never cascade into a whole + subtree. The cutoff is compared per branch by each branch's own + policy, which is what lets a monthly branch and a yearly branch answer + the same cutoff differently. + + Args: + plan: Tree description. + cutoff: Exclusive retention boundary (date or datetime). + dry_run: When true (the default), render the drops without + executing them. + through: Optional wider horizon for expected-spec expansion. + concurrent_detach: On PostgreSQL 14+, detach each leaf with + ``DETACH PARTITION CONCURRENTLY`` before dropping its table. + Lower versions keep the plain ``DROP TABLE`` path. + + Returns: + A report whose ``operations`` lists the (possibly not executed) + drops. + """ + infos = self._infos(plan) + specs = self._expected(plan, through=through if through is not None else cutoff) + candidates = self._drop_candidates(plan, cutoff, infos, through) + operations = tuple( + PartitionOperation( + action=action, + name=spec.name, + parent=spec.parent, + sql=sql, + level=spec.level, + ) + for spec in candidates + for action, sql, _ in self._drop_statements( + plan, spec, concurrent_detach + ) + ) + if not dry_run: + concurrent = ( + concurrent_detach and self.dialect.supports_concurrent_detach() + ) + if concurrent: + # DETACH ... CONCURRENTLY cannot run inside a transaction block, + # so the concurrent path executes statement by statement. + for unit in self._units(plan, candidates): + for spec in unit: + for _, sql, params in self._drop_statements(plan, spec, True): + self._execute(sql, params) + else: + def drop(spec: PartitionSpec) -> None: + for _, sql, params in self._drop_statements(plan, spec, False): + self._execute(sql, params) + + self._run_units(plan, self._units(plan, candidates), drop) + infos = self._infos(plan) + specs = self._without(specs, [spec.name for spec in candidates]) + return self._report(plan, infos, specs, through, operations, dry_run=dry_run) + + def drop_before_many( + self, + plans: Optional[Any] = None, + *, + pairs: Optional[Any] = None, + cutoff: Any, + dry_run: bool = True, + through: Optional[Any] = None, + concurrent_detach: bool = False, + ) -> PartitionBatchReport: + """Drop every leaf ending at or before *cutoff* across several plans. + + The drops run on one transaction. ``DETACH PARTITION CONCURRENTLY`` + cannot run inside a transaction, so the concurrent path is used only + when the batch holds a single plan; a multi-plan batch records that + limitation as an issue and keeps the transactional plain-drop path. + + Args: + plans: Independent plans to retain. + pairs: ``(pair_name, left_plan, right_plan)`` triples to retain and + compare. + cutoff: Exclusive retention boundary (date or datetime). + dry_run: When true (the default), render drops without executing. + through: Optional wider horizon for expected-spec expansion. + concurrent_detach: Request detach-then-drop on PostgreSQL 14+. + + Returns: + A batch report keyed by target, plus the paired drift. A dry run + describes the pre-run state; a real run describes the post-state. + """ + pair_entries, batch_plans = self._resolve_batch(plans, pairs) + supports_concurrent = self.dialect.supports_concurrent_detach() + concurrent = bool(concurrent_detach) and supports_concurrent + batch_issues: List[str] = [] + if concurrent and len(batch_plans) > 1: + concurrent = False + batch_issues.append( + "concurrent detach requires a single plan; batch has %d" + % len(batch_plans) + ) + prepared = [] + for plan in batch_plans: + infos = self._infos(plan) + prepared.append( + (plan, infos) + + self._drop_batch_work(plan, infos, cutoff, through, concurrent) + ) + if not dry_run: + if concurrent: + # DETACH ... CONCURRENTLY cannot run inside a transaction block. + for _, _, _, _, _, statements in prepared: + for sql, params in statements: + self._execute(sql, params) + else: + with self.backend.transaction(): + for _, _, _, _, _, statements in prepared: + for sql, params in statements: + self._execute(sql, params) + reports: Dict[str, PartitionReport] = {} + for plan, infos, specs, candidates, operations, _ in prepared: + if dry_run: + reports[plan.parent] = self._report( + plan, infos, specs, through, operations, dry_run=True + ) + continue + fresh = self._infos(plan) + remaining = self._without(specs, [spec.name for spec in candidates]) + reports[plan.parent] = self._report( + plan, fresh, remaining, through, operations, dry_run=False + ) + differences, issues = self._paired_report(pair_entries, reports) + return self._batch_report( + pair_entries, reports, differences, tuple(batch_issues) + issues + ) + + def drop_branch( + self, + plan: PostgresPartitionPlan, + branch: PartitionBranch, + dry_run: bool = False, + ) -> PartitionReport: + """Drop a whole first-level branch and its subpartitions (destructive by default). + + Pass ``dry_run=True`` to render the drop without executing it. This is + the explicit operation reserved for removing a branch; ``drop_before`` + never removes a whole branch implicitly. + + Args: + plan: Tree description. + branch: The first-level branch instance declared by the plan. + dry_run: When true, render the drop without executing it; defaults + to False, which executes the drop. + + Returns: + The report after (or before, under dry run) the drop. + """ + infos = self._infos(plan) + index = plan.branches.index(branch) + name = plan.branch_name(index, branch) + if name not in self._actual_names(infos): + return self._report(plan, infos, self._safe_expected(plan), None) + sql, params = self._drop_statement(plan, name, cascade=True) + operation = PartitionOperation("drop_branch", name, plan.parent, sql, 1) + if not dry_run: + with self.backend.transaction(): + self._execute(sql, params) + infos = self._infos(plan) + specs = self._safe_expected(plan) + specs = self._without(specs, [name] + [spec.name for spec in specs if spec.parent == name]) + return self._report(plan, infos, specs, None, (operation,), dry_run=dry_run) + + def validate( + self, + plan: PostgresPartitionPlan, + through: Optional[Any] = None, + ) -> PartitionReport: + """Report continuity and structure problems without changing anything. + + Checks gaps and overlaps in the plan's generated RANGE boundaries, + parent/strategy mismatches against the catalog, missing partitions and + orphans. It never repairs a gap; ``ensure`` is the explicit repair. + + Args: + plan: Tree description. + through: Exclusive horizon used to expand generated RANGE policies. + + Returns: + A report whose ``issues`` lists every problem found. + """ + return self.list(plan, through) + + +class AsyncPostgresPartitionMaintenance(_PartitionMaintenanceBase): + """Asynchronous PostgreSQL partition maintenance executor. + + The API mirrors :class:`PostgresPartitionMaintenance` exactly; every method + is a coroutine and the DDL transaction units are PostgreSQL's own. + """ + + async def _fetch_rows(self, sql: str, params: Tuple[Any, ...]): + return await self.backend.fetch_all(sql, params) + + async def _execute(self, sql: str, params: Tuple[Any, ...]) -> None: + await self.backend.execute(sql, params) + + async def _infos(self, plan: PostgresPartitionPlan) -> Tuple[PartitionInfo, ...]: + sql, params = self._tree_expression(plan).to_sql() + rows = await self._fetch_rows(sql, params) + return self._parse_rows(rows) + + async def _run_units(self, plan, units, callback) -> None: + for unit in units: + async with self.backend.transaction(): + for spec in unit: + await callback(spec) + + async def list( + self, + plan: PostgresPartitionPlan, + through: Optional[Any] = None, + ) -> PartitionReport: + """Read the live tree and compare it with the plan asynchronously.""" + infos = await self._infos(plan) + specs = self._safe_expected(plan, through) + report = self._report(plan, infos, specs, through) + if not plan.is_fully_bounded(through=through): + report = replace( + report, issues=report.issues + ("plan has no finite horizon",) + ) + return report + + async def list_many( + self, + plans: Optional[Any] = None, + *, + pairs: Optional[Any] = None, + ) -> PartitionBatchReport: + """Read several plans asynchronously and report paired drift.""" + pair_entries, batch_plans = self._resolve_batch(plans, pairs) + reports: Dict[str, PartitionReport] = {} + for plan in batch_plans: + reports[plan.parent] = await self.list(plan) + differences, issues = self._paired_report(pair_entries, reports) + return self._batch_report(pair_entries, reports, differences, issues) + + async def ensure( + self, + plan: PostgresPartitionPlan, + through: Optional[Any] = None, + if_not_exists: bool = True, + ) -> PartitionReport: + """Create every missing partition asynchronously.""" + if not plan.is_fully_bounded(through=through): + raise ValueError("plan has no finite horizon") + infos = await self._infos(plan) + if not infos: + raise ValueError("partitioned parent %s does not exist" % plan.parent) + specs = self._expected(plan, through) + actual = self._actual_names(infos) + missing = tuple( + spec for spec in specs if self._needs_partition(spec, actual) + ) + operations: List[PartitionOperation] = [] + + async def create(spec: PartitionSpec) -> None: + sql, params = self._create_statement(plan, spec, if_not_exists) + await self._execute(sql, params) + operations.append( + PartitionOperation("create", spec.name, spec.parent, sql, spec.level) + ) + + await self._run_units(plan, self._units(plan, missing), create) + report = await self.list(plan, through) + return replace(report, operations=tuple(operations), dry_run=False) + + async def ensure_many( + self, + plans: Optional[Any] = None, + *, + pairs: Optional[Any] = None, + through: Optional[Any] = None, + if_not_exists: bool = True, + ) -> PartitionBatchReport: + """Create every missing partition for several plans asynchronously. + + The whole batch runs in one transaction, matching + :meth:`PostgresPartitionMaintenance.ensure_many`. + """ + pair_entries, batch_plans = self._resolve_batch(plans, pairs) + prepared = [] + for plan in batch_plans: + infos = await self._infos(plan) + prepared.append( + (plan,) + self._ensure_batch_work( + plan, infos, through, if_not_exists + ) + ) + async with self.backend.transaction(): + for _, _, _, statements in prepared: + for sql, params in statements: + await self._execute(sql, params) + reports: Dict[str, PartitionReport] = {} + for plan, specs, operations, _ in prepared: + infos = await self._infos(plan) + reports[plan.parent] = self._report( + plan, infos, specs, through, operations, dry_run=False + ) + differences, issues = self._paired_report(pair_entries, reports) + return self._batch_report(pair_entries, reports, differences, issues) + + async def drop_before( + self, + plan: PostgresPartitionPlan, + cutoff: Any, + dry_run: bool = True, + through: Optional[Any] = None, + concurrent_detach: bool = False, + ) -> PartitionReport: + """Drop leaves ending at or before *cutoff* asynchronously.""" + infos = await self._infos(plan) + specs = self._expected(plan, through=through if through is not None else cutoff) + candidates = self._drop_candidates(plan, cutoff, infos, through) + operations = tuple( + PartitionOperation( + action=action, + name=spec.name, + parent=spec.parent, + sql=sql, + level=spec.level, + ) + for spec in candidates + for action, sql, _ in self._drop_statements( + plan, spec, concurrent_detach + ) + ) + if not dry_run: + concurrent = ( + concurrent_detach and self.dialect.supports_concurrent_detach() + ) + if concurrent: + # DETACH ... CONCURRENTLY cannot run inside a transaction block. + for unit in self._units(plan, candidates): + for spec in unit: + for _, sql, params in self._drop_statements(plan, spec, True): + await self._execute(sql, params) + else: + async def drop(spec: PartitionSpec) -> None: + for _, sql, params in self._drop_statements(plan, spec, False): + await self._execute(sql, params) + + await self._run_units(plan, self._units(plan, candidates), drop) + infos = await self._infos(plan) + specs = self._without(specs, [spec.name for spec in candidates]) + return self._report(plan, infos, specs, through, operations, dry_run=dry_run) + + async def drop_before_many( + self, + plans: Optional[Any] = None, + *, + pairs: Optional[Any] = None, + cutoff: Any, + dry_run: bool = True, + through: Optional[Any] = None, + concurrent_detach: bool = False, + ) -> PartitionBatchReport: + """Drop every leaf ending at or before *cutoff* asynchronously. + + This is the asynchronous counterpart of + :meth:`PostgresPartitionMaintenance.drop_before_many`. + """ + pair_entries, batch_plans = self._resolve_batch(plans, pairs) + supports_concurrent = self.dialect.supports_concurrent_detach() + concurrent = bool(concurrent_detach) and supports_concurrent + batch_issues: List[str] = [] + if concurrent and len(batch_plans) > 1: + concurrent = False + batch_issues.append( + "concurrent detach requires a single plan; batch has %d" + % len(batch_plans) + ) + prepared = [] + for plan in batch_plans: + infos = await self._infos(plan) + prepared.append( + (plan, infos) + + self._drop_batch_work(plan, infos, cutoff, through, concurrent) + ) + if not dry_run: + if concurrent: + # DETACH ... CONCURRENTLY cannot run inside a transaction block. + for _, _, _, _, _, statements in prepared: + for sql, params in statements: + await self._execute(sql, params) + else: + async with self.backend.transaction(): + for _, _, _, _, _, statements in prepared: + for sql, params in statements: + await self._execute(sql, params) + reports: Dict[str, PartitionReport] = {} + for plan, infos, specs, candidates, operations, _ in prepared: + if dry_run: + reports[plan.parent] = self._report( + plan, infos, specs, through, operations, dry_run=True + ) + continue + fresh = await self._infos(plan) + remaining = self._without(specs, [spec.name for spec in candidates]) + reports[plan.parent] = self._report( + plan, fresh, remaining, through, operations, dry_run=False + ) + differences, issues = self._paired_report(pair_entries, reports) + return self._batch_report( + pair_entries, reports, differences, tuple(batch_issues) + issues + ) + + async def drop_branch( + self, + plan: PostgresPartitionPlan, + branch: PartitionBranch, + dry_run: bool = False, + ) -> PartitionReport: + """Drop a whole first-level branch asynchronously (destructive by default). + + The dropped branch takes its subpartitions with it. Pass + ``dry_run=True`` to render the drop without executing it. This is + the explicit operation reserved for removing a branch; ``drop_before`` + never removes a whole branch implicitly. + + Args: + plan: Tree description. + branch: The first-level branch instance declared by the plan. + dry_run: When true, render the drop without executing it; defaults + to False, which executes the drop. + + Returns: + The report after (or before, under dry run) the drop. + """ + infos = await self._infos(plan) + index = plan.branches.index(branch) + name = plan.branch_name(index, branch) + if name not in self._actual_names(infos): + return self._report(plan, infos, self._safe_expected(plan), None) + sql, params = self._drop_statement(plan, name, cascade=True) + operation = PartitionOperation("drop_branch", name, plan.parent, sql, 1) + if not dry_run: + async with self.backend.transaction(): + await self._execute(sql, params) + infos = await self._infos(plan) + specs = self._safe_expected(plan) + specs = self._without(specs, [name] + [spec.name for spec in specs if spec.parent == name]) + return self._report(plan, infos, specs, None, (operation,), dry_run=dry_run) + + async def validate( + self, + plan: PostgresPartitionPlan, + through: Optional[Any] = None, + ) -> PartitionReport: + """Report continuity and structure problems asynchronously.""" + return await self.list(plan, through) diff --git a/src/rhosocial/activerecord/backend/impl/postgres/partition/plan.py b/src/rhosocial/activerecord/backend/impl/postgres/partition/plan.py new file mode 100644 index 00000000..19765c2a --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/partition/plan.py @@ -0,0 +1,670 @@ +# src/rhosocial/activerecord/backend/impl/postgres/partition/plan.py +"""Owner-agnostic partition plans for PostgreSQL. + +ActiveRecord deliberately does not model partitioning (see the core plan +``2026-11-02/partitioned-model-support.md``): a model declares the primary-key +shape a partitioned table needs, and the backend owns the tree. This module is +the backend's description of a tree: + + Parent(HASH, tenant_bucket, modulus=4) + |-- p0 (RANGE, created_at, monthly) + |-- p1 (RANGE, created_at, quarterly) + |-- p2 (RANGE, created_at, yearly) + `-- p3 (RANGE, created_at, explicit 15-day bounds) + +PostgreSQL allows each branch of a HASH-partitioned parent to choose its own +child strategy, so the plan is a tree of policies, not one flat strategy. A +plan carries table names, columns and policies only; it carries no application +names, no retention default and no application vocabulary. Those belong to the +caller. + +Policies describe the partitions *below* their parent. ``RangePolicy``, +``ListPolicy`` and ``HashPolicy`` each generate concrete boundaries and +labels; naming and DDL rendering are the maintenance executor's job. +""" + +import re +from dataclasses import dataclass +from datetime import date, datetime +from typing import Any, Dict, Iterator, Optional, Tuple, Union + +from .bounds import default_label, iter_range_bounds, normalize_interval + +__all__ = [ + "HashBranch", + "HashPolicy", + "ListBranch", + "ListPolicy", + "PartitionPolicy", + "PartitionSpec", + "PostgresPartitionPlan", + "RangeBranch", + "RangePolicy", +] + +_METHODS = ("RANGE", "LIST", "HASH") + + +def _as_columns(value: Any) -> Tuple[str, ...]: + """Normalise a column declaration to a tuple of non-empty names.""" + if isinstance(value, str): + columns = (value,) + elif isinstance(value, (tuple, list)): + columns = tuple(value) + else: + raise TypeError("columns must be a string or a tuple of strings") + if not columns: + raise ValueError("columns must not be empty") + for column in columns: + if not isinstance(column, str) or not column.strip(): + raise ValueError("every column must be a non-empty string") + return columns + + +def _normalise_method(method: Any) -> str: + if isinstance(method, str): + normalized = method.upper() + if normalized in _METHODS: + return normalized + raise ValueError("method must be one of %s, got %r" % (", ".join(_METHODS), method)) + + +def _label_from_values(values: Tuple[Any, ...]) -> str: + text = "_".join(str(value) for value in values) + label = re.sub(r"[^0-9A-Za-z]+", "_", text).strip("_") + return label or "value" + + +def _render_label(pattern: str, start: Any, end: Any) -> str: + return pattern.format( + start=start, + end=end, + year=start.year, + month=start.month, + day=start.day, + quarter=(start.month - 1) // 3 + 1, + ) + + +@dataclass(frozen=True) +class RangePolicy: + """Generate RANGE partitions with half-open ``[start, end)`` bounds. + + Either *bounds* supplies every boundary explicitly, or *start* plus + *interval* generates them. A generated policy needs a horizon: *count*, or + a ``through`` argument on the maintenance call. The same policy object can + serve one flat monthly table or one branch of a HASH tree (each branch + advancing at its own granularity). + + Attributes: + columns: Partition key columns below the policy's parent. + start: First boundary when *bounds* is not given. + interval: Step between generated boundaries; a named unit, a + ``" "`` spelling, or an ``(amount, unit)`` pair. + count: Maximum number of generated boundaries. + bounds: Explicit ``(start, end)`` pairs, mutually exclusive with the + generated form. + label_pattern: Optional ``str.format`` pattern for the name suffix; + receives ``start``, ``end``, ``year``, ``month``, ``day`` and + ``quarter``. + """ + + columns: Union[str, Tuple[str, ...]] = "created_at" + start: Optional[Any] = None + interval: Any = "month" + count: Optional[int] = None + bounds: Optional[Tuple[Tuple[Any, Any], ...]] = None + label_pattern: Optional[str] = None + + def __post_init__(self) -> None: + object.__setattr__(self, "columns", _as_columns(self.columns)) + if self.bounds is not None: + pairs = tuple(tuple(pair) for pair in self.bounds) + if self.start is not None: + raise ValueError("start and bounds are mutually exclusive") + else: + pairs = None + if self.start is None: + raise ValueError("RangePolicy requires start or bounds") + if not isinstance(self.start, (date, datetime)): + raise TypeError("RangePolicy start must be date or datetime") + normalize_interval(self.interval) + if self.count is not None: + if isinstance(self.count, bool) or not isinstance(self.count, int) or self.count <= 0: + raise ValueError("count must be a positive integer") + if pairs is not None: + if not pairs: + raise ValueError("bounds must not be empty") + for pair in pairs: + if len(pair) != 2: + raise ValueError("every bounds entry must be (start, end)") + if not isinstance(pair[0], (date, datetime)) or not isinstance( + pair[1], (date, datetime) + ): + raise TypeError("bound values must be date or datetime") + if pair[0] >= pair[1]: + raise ValueError("bound start must be before end") + object.__setattr__(self, "bounds", pairs) + + @property + def method(self) -> str: + """The partitioning strategy this policy generates.""" + return "RANGE" + + def iter_bounds(self, through: Optional[Any] = None) -> Iterator[Tuple[Any, Any]]: + """Yield the policy's ``(start, end)`` boundaries. + + Explicit bounds are yielded as declared. Generated bounds stop at + *count* or at *through*, whichever comes first. + + Raises: + ValueError: when the policy has no horizon, or when *through* is + not after the first boundary. + """ + if self.bounds is not None: + for pair in self.bounds: + yield pair[0], pair[1] + return + yield from iter_range_bounds( + self.start, + self.interval, + count=self.count, + through=through, + ) + + def label_for(self, start: Any, end: Any) -> str: + """Return the partition-name suffix for one generated boundary.""" + if self.label_pattern: + return _render_label(self.label_pattern, start, end) + if self.bounds is not None: + return start.strftime("%Y_%m_%d") + return default_label(start, self.interval) + + +@dataclass(frozen=True) +class ListPolicy: + """Generate LIST partitions from explicit value groups. + + Attributes: + columns: Partition key column (PostgreSQL LIST accepts exactly one). + groups: One tuple of values per generated partition. + labels: Optional name suffix per group, same length as *groups*. + """ + + columns: Union[str, Tuple[str, ...]] = "status" + groups: Tuple[Tuple[Any, ...], ...] = () + labels: Optional[Tuple[str, ...]] = None + + def __post_init__(self) -> None: + object.__setattr__(self, "columns", _as_columns(self.columns)) + groups = tuple(tuple(group) for group in self.groups) + if not groups: + raise ValueError("ListPolicy requires at least one value group") + for group in groups: + if not group: + raise ValueError("every value group must be non-empty") + object.__setattr__(self, "groups", groups) + if self.labels is not None: + labels = tuple(self.labels) + if len(labels) != len(groups): + raise ValueError("labels must have one entry per value group") + for label in labels: + if not isinstance(label, str) or not label.strip(): + raise ValueError("every label must be a non-empty string") + object.__setattr__(self, "labels", labels) + + @property + def method(self) -> str: + """The partitioning strategy this policy generates.""" + return "LIST" + + def iter_groups(self) -> Iterator[Tuple[Tuple[Any, ...], str]]: + """Yield ``(values, label)`` for every generated partition.""" + for index, values in enumerate(self.groups): + if self.labels is not None: + label = self.labels[index] + else: + label = _label_from_values(values) + yield values, label + + +@dataclass(frozen=True) +class HashPolicy: + """Generate HASH partitions for a fixed modulus. + + Attributes: + columns: Partition key columns. + modulus: Number of hash buckets. + remainders: Buckets to generate; all buckets when omitted. + """ + + columns: Union[str, Tuple[str, ...]] = "bucket" + modulus: int = 1 + remainders: Optional[Tuple[int, ...]] = None + + def __post_init__(self) -> None: + object.__setattr__(self, "columns", _as_columns(self.columns)) + if isinstance(self.modulus, bool) or not isinstance(self.modulus, int): + raise TypeError("modulus must be an integer") + if self.modulus <= 0: + raise ValueError("modulus must be a positive integer") + if self.remainders is not None: + remainders = tuple(self.remainders) + if not remainders: + raise ValueError("remainders must not be empty") + for remainder in remainders: + if isinstance(remainder, bool) or not isinstance(remainder, int): + raise TypeError("remainder must be an integer") + if not 0 <= remainder < self.modulus: + raise ValueError("remainder must satisfy 0 <= remainder < modulus") + if len(set(remainders)) != len(remainders): + raise ValueError("remainders must be unique") + object.__setattr__(self, "remainders", remainders) + + @property + def method(self) -> str: + """The partitioning strategy this policy generates.""" + return "HASH" + + def iter_remainders(self) -> Iterator[int]: + """Yield the selected remainders in ascending order.""" + if self.remainders is not None: + yield from self.remainders + return + for remainder in range(self.modulus): + yield remainder + + +#: A policy is any of the three strategy generators. +PartitionPolicy = Union[RangePolicy, ListPolicy, HashPolicy] + + +@dataclass(frozen=True) +class HashBranch: + """A first-level partition of a HASH-partitioned parent. + + Attributes: + remainder: Hash bucket this branch owns. + child: Optional policy for partitions below this branch. + name: Optional branch name suffix, substituted into ``name_pattern``; + the default is ``p``, giving ``_p``. + """ + + remainder: int + child: Optional[PartitionPolicy] = None + name: Optional[str] = None + + def __post_init__(self) -> None: + if isinstance(self.remainder, bool) or not isinstance(self.remainder, int): + raise TypeError("remainder must be an integer") + if self.remainder < 0: + raise ValueError("remainder must not be negative") + if self.name is not None and not self.name.strip(): + raise ValueError("name must be a non-empty string") + + +@dataclass(frozen=True) +class RangeBranch: + """A first-level RANGE partition, independent of a policy horizon. + + Attributes: + start: Inclusive lower bound. + end: Exclusive upper bound. + child: Optional policy for partitions below this branch. + name: Optional branch name suffix, substituted into ``name_pattern``. + label: Optional name suffix instead of the derived date label. + """ + + start: Any + end: Any + child: Optional[PartitionPolicy] = None + name: Optional[str] = None + label: Optional[str] = None + + def __post_init__(self) -> None: + if not isinstance(self.start, (date, datetime)) or not isinstance( + self.end, (date, datetime) + ): + raise TypeError("RangeBranch bounds must be date or datetime") + if self.start >= self.end: + raise ValueError("RangeBranch start must be before end") + if self.name is not None and not self.name.strip(): + raise ValueError("name must be a non-empty string") + if self.label is not None and not self.label.strip(): + raise ValueError("label must be a non-empty string") + + +@dataclass(frozen=True) +class ListBranch: + """A first-level LIST partition, independent of a policy horizon. + + Attributes: + values: Values owned by this branch. + child: Optional policy for partitions below this branch. + name: Optional branch name suffix, substituted into ``name_pattern``. + label: Optional name suffix instead of the derived value label. + """ + + values: Tuple[Any, ...] + child: Optional[PartitionPolicy] = None + name: Optional[str] = None + label: Optional[str] = None + + def __post_init__(self) -> None: + values = tuple(self.values) + if not values: + raise ValueError("ListBranch values must not be empty") + object.__setattr__(self, "values", values) + if self.name is not None and not self.name.strip(): + raise ValueError("name must be a non-empty string") + if self.label is not None and not self.label.strip(): + raise ValueError("label must be a non-empty string") + + +#: A first-level partition of any parent strategy. +PartitionBranch = Union[HashBranch, RangeBranch, ListBranch] + + +@dataclass(frozen=True) +class PartitionSpec: + """One concrete partition the plan asks PostgreSQL to own. + + A spec is derived data: the executor turns it into ``CREATE TABLE ... + PARTITION OF`` and compares it with the catalog. ``child_method`` / + ``child_columns`` describe the clause attached to a branch that is itself + partitioned; ``range_end`` is the exclusive upper bound of a RANGE leaf, + which is what ``drop_before`` compares against a cutoff. + """ + + name: str + parent: str + method: str + columns: Tuple[str, ...] + bounds: Dict[str, Any] + level: int + is_leaf: bool + child_method: Optional[str] = None + child_columns: Tuple[str, ...] = () + range_start: Optional[Any] = None + range_end: Optional[Any] = None + label: Optional[str] = None + + +@dataclass(frozen=True) +class PostgresPartitionPlan: + """A PostgreSQL partition tree, described by policies rather than names. + + Exactly one of *policy* and *branches* is supplied: + + * *policy* generates every partition directly below the parent. This is the + flat single-table shape -- a RANGE parent with one monthly policy -- and + is also how a HASH or LIST parent gets all of its first-level partitions. + * *branches* lists first-level partitions explicitly, each optionally + carrying its own child policy. Branches are how a PostgreSQL HASH parent + gives different RANGE granularities to different hash buckets. + + Attributes: + parent: Partitioned parent table name. + method: Strategy the parent uses: ``RANGE``, ``LIST`` or ``HASH``. + columns: Parent partition key columns. + modulus: HASH bucket count when *branches* holds ``HashBranch`` values. + branches: Explicit first-level partitions. + policy: Auto-generated first-level partitions. + schema: Optional schema shared by the parent and every partition. + name_pattern: ``str.format`` pattern for generated names; must contain + ``{suffix}`` and normally also ``{parent}``. + """ + + parent: str + method: str + columns: Union[str, Tuple[str, ...]] = () + modulus: Optional[int] = None + branches: Tuple[PartitionBranch, ...] = () + policy: Optional[PartitionPolicy] = None + schema: Optional[str] = None + name_pattern: str = "{parent}_{suffix}" + + def __post_init__(self) -> None: + if not isinstance(self.parent, str) or not self.parent.strip(): + raise ValueError("parent must be a non-empty string") + object.__setattr__(self, "method", _normalise_method(self.method)) + object.__setattr__(self, "columns", _as_columns(self.columns)) + if not self.name_pattern: + raise ValueError("name_pattern must not be empty") + if "{suffix}" not in self.name_pattern: + raise ValueError("name_pattern must contain {suffix}") + if self.policy is not None and self.branches: + raise ValueError("policy and branches are mutually exclusive") + if self.schema is not None and not self.schema.strip(): + raise ValueError("schema must be a non-empty string") + + branches = tuple(self.branches) + for index, branch in enumerate(branches): + self._check_branch(index, branch) + object.__setattr__(self, "branches", branches) + + if self.policy is not None: + self._check_flat_policy(self.policy) + self._check_policy(self.policy) + + if self.method == "HASH": + if self.policy is not None: + if not isinstance(self.policy, HashPolicy): + raise ValueError("a HASH parent's policy must be a HashPolicy") + elif branches: + if self.modulus is None: + raise ValueError("a HASH parent with branches requires modulus") + if isinstance(self.modulus, bool) or not isinstance(self.modulus, int): + raise TypeError("modulus must be an integer") + if self.modulus <= 0: + raise ValueError("modulus must be a positive integer") + for branch in branches: + if branch.remainder >= self.modulus: + raise ValueError("HashBranch remainder must be less than modulus") + elif self.modulus is not None: + raise ValueError("modulus is only meaningful for a HASH parent") + + def _check_flat_policy(self, policy: PartitionPolicy) -> None: + expected = { + "RANGE": RangePolicy, + "LIST": ListPolicy, + "HASH": HashPolicy, + }[self.method] + if not isinstance(policy, expected): + raise ValueError( + "a %s parent's policy must be a %s" % (self.method, expected.__name__) + ) + + @staticmethod + def _check_policy(policy: PartitionPolicy) -> None: + """Validate a policy regardless of which parent level owns it.""" + if not isinstance(policy, (RangePolicy, ListPolicy, HashPolicy)): + raise TypeError("child must be a RangePolicy, ListPolicy or HashPolicy") + if isinstance(policy, HashPolicy) and policy.modulus < 1: + raise ValueError("HashPolicy modulus must be positive") + + def _check_branch(self, index: int, branch: PartitionBranch) -> None: + if self.method == "HASH": + if not isinstance(branch, HashBranch): + raise ValueError("a HASH parent's branches must be HashBranch values") + elif self.method == "RANGE": + if not isinstance(branch, RangeBranch): + raise ValueError("a RANGE parent's branches must be RangeBranch values") + else: + if not isinstance(branch, ListBranch): + raise ValueError("a LIST parent's branches must be ListBranch values") + if branch.child is not None: + self._check_policy(branch.child) + + def _name(self, suffix: str, **placeholders: Any) -> str: + return self.name_pattern.format( + parent=self.parent, suffix=suffix, **placeholders + ) + + def branch_suffix(self, index: int, branch: PartitionBranch) -> str: + """Return the name suffix reserved for a first-level branch.""" + if branch.name is not None: + return branch.name + if isinstance(branch, HashBranch): + return "p%d" % branch.remainder + if branch.label is not None: + return branch.label + if isinstance(branch, RangeBranch): + return branch.start.strftime("%Y_%m_%d") + return _label_from_values(branch.values) + + def branch_name(self, index: int, branch: PartitionBranch) -> str: + """Return the final table name of a first-level branch. + + ``name`` is a suffix, not a literal table name, so a caller that renames + a branch still gets the plan's ``name_pattern`` and its child leaves + compose under the same prefix. + """ + suffix = self.branch_suffix(index, branch) + return self._name(suffix, index=index, remainder=getattr(branch, "remainder", None)) + + def _policy_specs( + self, + policy: PartitionPolicy, + parent: str, + level: int, + prefix: Optional[str], + through: Optional[Any], + ) -> Iterator[PartitionSpec]: + branch_placeholders = {"index": None, "remainder": None} + if isinstance(policy, RangePolicy): + for start, end in policy.iter_bounds(through=through): + label = policy.label_for(start, end) + suffix = label if prefix is None else "%s_%s" % (prefix, label) + yield PartitionSpec( + name=self._name(suffix, **branch_placeholders), + parent=parent, + method="RANGE", + columns=policy.columns, + bounds={"from": start, "to": end}, + level=level, + is_leaf=True, + range_start=start, + range_end=end, + label=label, + ) + elif isinstance(policy, ListPolicy): + for values, label in policy.iter_groups(): + suffix = label if prefix is None else "%s_%s" % (prefix, label) + yield PartitionSpec( + name=self._name(suffix, **branch_placeholders), + parent=parent, + method="LIST", + columns=policy.columns, + bounds={"values": values}, + level=level, + is_leaf=True, + label=label, + ) + else: + for remainder in policy.iter_remainders(): + label = "p%d" % remainder + suffix = label if prefix is None else "%s_%s" % (prefix, label) + yield PartitionSpec( + name=self._name(suffix, **branch_placeholders), + parent=parent, + method="HASH", + columns=policy.columns, + bounds={"modulus": policy.modulus, "remainder": remainder}, + level=level, + is_leaf=True, + label=label, + ) + + def _branch_spec( + self, + index: int, + branch: PartitionBranch, + ) -> PartitionSpec: + name = self.branch_name(index, branch) + suffix = self.branch_suffix(index, branch) + child_method = None + child_columns: Tuple[str, ...] = () + if branch.child is not None: + child_method = branch.child.method + child_columns = branch.child.columns + bounds: Dict[str, Any] + if isinstance(branch, HashBranch): + bounds = {"modulus": self.modulus, "remainder": branch.remainder} + elif isinstance(branch, RangeBranch): + bounds = {"from": branch.start, "to": branch.end} + else: + bounds = {"values": branch.values} + return PartitionSpec( + name=name, + parent=self.parent, + method=self.method, + columns=self.columns, + bounds=bounds, + level=1, + is_leaf=branch.child is None, + child_method=child_method, + child_columns=child_columns, + range_start=branch.start if isinstance(branch, RangeBranch) else None, + range_end=branch.end if isinstance(branch, RangeBranch) else None, + label=suffix, + ) + + def iter_specs(self, through: Optional[Any] = None) -> Iterator[PartitionSpec]: + """Yield every partition the plan describes. + + First-level branches are yielded before their children, in declaration + order, so an executor can create them in yielded order. Names are + checked for uniqueness here rather than at execution time. + + Raises: + ValueError: if two generated partitions would share a name, or if + a generated RANGE policy has no horizon. + """ + yielded: Dict[str, PartitionSpec] = {} + + def emit(spec: PartitionSpec) -> PartitionSpec: + if spec.name in yielded: + raise ValueError("plan generates duplicate partition name %r" % spec.name) + yielded[spec.name] = spec + return spec + + if self.policy is not None: + for spec in self._policy_specs(self.policy, self.parent, 1, None, through): + yield emit(spec) + return + + if not self.branches: + return + + for index, branch in enumerate(self.branches): + spec = self._branch_spec(index, branch) + yield emit(spec) + if branch.child is not None: + prefix = spec.label + for child in self._policy_specs( + branch.child, spec.name, 2, prefix, through + ): + yield emit(child) + + def is_fully_bounded(self, through: Optional[Any] = None) -> bool: + """Whether every policy has a finite horizon. + + A plan whose RANGE policy has neither explicit bounds nor *count*, and + whose call supplies no *through*, cannot say which partitions are + expected in the future. ``list`` still reports what exists, but it does + not label extra partitions orphans in that case. + """ + if through is not None: + return True + policies = [] + if self.policy is not None: + policies.append(self.policy) + for branch in self.branches: + if branch.child is not None: + policies.append(branch.child) + for policy in policies: + if isinstance(policy, RangePolicy): + if policy.bounds is None and policy.count is None: + return False + return True diff --git a/src/rhosocial/activerecord/backend/impl/postgres/partition/report.py b/src/rhosocial/activerecord/backend/impl/postgres/partition/report.py new file mode 100644 index 00000000..448a7048 --- /dev/null +++ b/src/rhosocial/activerecord/backend/impl/postgres/partition/report.py @@ -0,0 +1,25 @@ +# src/rhosocial/activerecord/backend/impl/postgres/partition/report.py +"""PostgreSQL's partition reports are the core contract's reports. + +PostgreSQL is the backend whose report shape the contract was drawn from, so it +has no engine-specific row fields left: the tree row, the statement record, the +report envelope and the batch/paired view come from +:mod:`rhosocial.activerecord.backend.partition`. This module remains as the +import site the rest of this package already uses. +""" + +from rhosocial.activerecord.backend.partition import ( + PairedDifference, + PartitionBatchReport, + PartitionInfo, + PartitionOperation, + PartitionReport, +) + +__all__ = [ + "PairedDifference", + "PartitionBatchReport", + "PartitionInfo", + "PartitionOperation", + "PartitionReport", +] diff --git a/src/rhosocial/activerecord/backend/impl/postgres/protocols/ddl/partition.py b/src/rhosocial/activerecord/backend/impl/postgres/protocols/ddl/partition.py index c52097a1..e152392b 100644 --- a/src/rhosocial/activerecord/backend/impl/postgres/protocols/ddl/partition.py +++ b/src/rhosocial/activerecord/backend/impl/postgres/protocols/ddl/partition.py @@ -16,6 +16,7 @@ PostgresDetachPartitionStatement, PostgresAttachPartitionStatement, PostgresPartitionMetadataQueryStatement, + PostgresPartitionTreeQueryStatement, ) @@ -208,3 +209,20 @@ def format_partition_metadata_query_statement( UnsupportedFeatureError: If PostgreSQL version < 10. """ ... + + def format_partition_tree_query_statement( + self, expr: "PostgresPartitionTreeQueryStatement" + ) -> Tuple[str, tuple]: + """Format the recursive pg_catalog query for a partition subtree. + + Starts at the root table named by ``expr.qualified_name`` and walks + ``pg_inherits`` to every descendant, carrying the server's bound text, + the child strategy, the tree level and optional size estimates. + + Args: + expr: PostgresPartitionTreeQueryStatement instance. + + Returns: + Tuple of (SQL string with one ``%s`` placeholder, params tuple). + """ + ... diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/partition_syntax_fixtures.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/partition_syntax_fixtures.py new file mode 100644 index 00000000..92cd37e7 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/partition_syntax_fixtures.py @@ -0,0 +1,631 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/partition_syntax_fixtures.py +"""Fixture table for PostgreSQL partition syntax families. + +Every case is an offline description of one PostgreSQL partitioning shape: +the plan (or a statement-only clause case), the exact specs the plan must +generate, the DDL fragments that must and must not appear, and -- where a +family has range leaves -- the ``drop_before`` selection the executor must +make. + +The matrix deliberately lives in fixtures rather than in test bodies: adding +a new PostgreSQL syntax family is one entry in :data:`CASES`, and the tests in +``test_partition_syntax_matrix.py`` apply the same checks to it. + +DEFAULT partitions are not represented here. The statement layer can render +``PARTITION OF ... DEFAULT`` (``partition_values={"default": True}``), but +``RangePolicy``/``ListPolicy``/``HashPolicy`` and therefore +``PostgresPartitionPlan`` have no way to describe an unbounded DEFAULT child, +so a plan-driven DEFAULT case would not exercise the plan API at all. It is +reported as unsupported rather than encoded as a misleading case. +""" + +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from typing import Any, Callable, Dict, Optional, Tuple + +import pytest + +from rhosocial.activerecord.backend.impl.postgres.expression.ddl.partition import ( + PostgresAttachPartitionStatement, + PostgresDetachPartitionStatement, +) +from rhosocial.activerecord.backend.impl.postgres.partition import ( + HashBranch, + HashPolicy, + ListBranch, + ListPolicy, + PostgresPartitionPlan, + RangeBranch, + RangePolicy, +) + +UTC = timezone.utc + + +def _dt(year: int, month: int, day: int) -> datetime: + """Build a timezone-aware UTC datetime used by every generated bound.""" + return datetime(year, month, day, tzinfo=UTC) + + +@dataclass(frozen=True) +class ExpectedSpecFacts: + """One expected ``PartitionSpec``, written as data instead of a guess.""" + + name: str + parent: str + level: int + is_leaf: bool + method: str + bounds: Dict[str, Any] + child_method: Optional[str] = None + child_columns: Tuple[str, ...] = () + + +@dataclass(frozen=True) +class ClauseStatement: + """One statement-only case: how to build it and the SQL it must render.""" + + kind: str + build: Callable[[Any], Any] + expected_sql: str + + +@dataclass(frozen=True) +class ClauseCase: + """Statement cases that have no partition plan (ATTACH/DETACH).""" + + statements: Tuple[ClauseStatement, ...] + concurrent_attach_rejected: bool = False + + +@dataclass(frozen=True) +class DropBeforeExpectation: + """The cutoff and the leaf names ``drop_before`` must select.""" + + cutoff: Any + candidates: Tuple[str, ...] + + +@dataclass(frozen=True) +class PartitionSyntaxCase: + """One fixtures-driven PostgreSQL partition syntax family. + + Attributes: + name: Stable pytest id. + description: What syntax family this case pins. + plan: Plan-driven case, or ``None`` for a statement-only clause case. + clause: Statement-only case, or ``None`` for a plan-driven case. + expected_specs: Exact specs the plan must generate, in yielded order. + required_fragments: Substrings every rendered DDL set must contain. + forbidden_fragments: Substrings no rendered DDL statement may contain. + drop_before: Optional retention-selection expectation. + branch_leaf_counts: ``(branch name, direct child leaf count)`` pairs + for multi-level cases; the branch shape is checked as well. + live_setup: Optional callable or SQL text for a future live test. + live_check: Optional callable or SQL text for a future live test. + """ + + name: str + description: str + plan: Optional[PostgresPartitionPlan] = None + clause: Optional[ClauseCase] = None + expected_specs: Tuple[ExpectedSpecFacts, ...] = () + required_fragments: Tuple[str, ...] = () + forbidden_fragments: Tuple[str, ...] = () + drop_before: Optional[DropBeforeExpectation] = None + branch_leaf_counts: Tuple[Tuple[str, int], ...] = () + live_setup: Optional[Any] = None + live_check: Optional[Any] = None + + def __post_init__(self) -> None: + if (self.plan is None) == (self.clause is None): + raise ValueError("exactly one of plan or clause must be supplied") + + +def _leaf(name: str, parent: str, method: str, bounds: Dict[str, Any], + level: int = 1) -> ExpectedSpecFacts: + return ExpectedSpecFacts( + name=name, parent=parent, level=level, is_leaf=True, + method=method, bounds=bounds, + ) + + +def _branch(name: str, parent: str, method: str, bounds: Dict[str, Any], + child_method: Optional[str] = None, + child_columns: Tuple[str, ...] = ()) -> ExpectedSpecFacts: + return ExpectedSpecFacts( + name=name, parent=parent, level=1, is_leaf=child_method is None, + method=method, bounds=bounds, child_method=child_method, + child_columns=child_columns, + ) + + +def _monthly_leaf_facts(parent: str, year: int, start_month: int, + count: int, level: int = 2) -> Tuple[ExpectedSpecFacts, ...]: + facts = [] + for offset in range(count): + index = (start_month - 1) + offset + current_year = year + index // 12 + current_month = index % 12 + 1 + start = _dt(current_year, current_month, 1) + if current_month == 12: + end = _dt(current_year + 1, 1, 1) + else: + end = _dt(current_year, current_month + 1, 1) + name = "%s_%d_%02d" % (parent, current_year, current_month) + facts.append(_leaf(name, parent, "RANGE", {"from": start, "to": end}, + level=level)) + return tuple(facts) + + +def _quarterly_leaf_facts(parent: str) -> Tuple[ExpectedSpecFacts, ...]: + pairs = ( + (_dt(2026, 1, 1), _dt(2026, 4, 1)), + (_dt(2026, 4, 1), _dt(2026, 7, 1)), + (_dt(2026, 7, 1), _dt(2026, 10, 1)), + (_dt(2026, 10, 1), _dt(2027, 1, 1)), + ) + facts = [] + for quarter, (start, end) in enumerate(pairs, start=1): + name = "%s_2026_q%d" % (parent, quarter) + facts.append(_leaf(name, parent, "RANGE", {"from": start, "to": end}, + level=2)) + return tuple(facts) + + +def _daily_leaf_facts(parent: str, start: datetime, + count: int) -> Tuple[ExpectedSpecFacts, ...]: + facts = [] + for offset in range(count): + day = start + timedelta(days=offset) + name = "%s_%s" % (parent, day.strftime("%Y_%m_%d")) + facts.append(_leaf(name, parent, "RANGE", + {"from": day, "to": day + timedelta(days=1)}, + level=2)) + return tuple(facts) + + +_JOBS_FLAT_MONTHLY = PostgresPartitionPlan( + parent="job_logs", + method="RANGE", + columns=("attempt_created_at",), + policy=RangePolicy( + columns=("attempt_created_at",), + start=_dt(2026, 9, 1), + interval="month", + count=2, + ), +) + +_AUDIT_FLAT_EXPLICIT = PostgresPartitionPlan( + parent="audit_events", + method="RANGE", + columns=("created_at",), + policy=RangePolicy( + columns=("created_at",), + bounds=( + (_dt(2026, 1, 1), _dt(2026, 1, 16)), + (_dt(2026, 1, 16), _dt(2026, 2, 1)), + ), + ), +) + +_EVENTS_FLAT_LIST = PostgresPartitionPlan( + parent="events", + method="LIST", + columns=("region",), + policy=ListPolicy( + columns=("region",), + groups=(("us", "ca"), ("eu",), ("apac",)), + ), +) + +_SESSIONS_FLAT_HASH = PostgresPartitionPlan( + parent="sessions", + method="HASH", + columns=("tenant_id",), + policy=HashPolicy( + columns=("tenant_id",), + modulus=4, + remainders=(0, 2), + ), +) + +_AUDIT_RANGE_BRANCHES = PostgresPartitionPlan( + parent="audit", + method="RANGE", + columns=("created_at",), + branches=( + RangeBranch( + _dt(2026, 1, 1), + _dt(2026, 7, 1), + child=RangePolicy( + columns=("created_at",), + start=_dt(2026, 1, 1), + interval="month", + count=2, + ), + name="h1", + ), + RangeBranch(_dt(2026, 7, 1), _dt(2027, 1, 1), name="h2"), + ), +) + +_ORDERS_LIST_BRANCHES = PostgresPartitionPlan( + parent="orders", + method="LIST", + columns=("status",), + branches=( + ListBranch( + ("paid", "shipped"), + child=ListPolicy( + columns=("region",), + groups=(("eu",), ("us",)), + ), + name="closed", + ), + ListBranch(("cancelled",), name="cancelled"), + ), +) + +_TELEMETRY_BRANCHES = ( + HashBranch( + 0, + RangePolicy( + columns=("created_at",), + start=_dt(2026, 1, 1), + interval="month", + count=12, + ), + ), + HashBranch( + 1, + RangePolicy( + columns=("created_at",), + start=_dt(2026, 1, 1), + interval="quarter", + count=4, + ), + ), + HashBranch( + 2, + RangePolicy( + columns=("created_at",), + start=_dt(2026, 2, 1), + interval="day", + count=28, + ), + ), +) + + +CASES = [ + PartitionSyntaxCase( + name="flat_range_generated", + description=( + "Flat RANGE parent with one monthly policy: two generated, " + "half-open [start, end) leaves." + ), + plan=_JOBS_FLAT_MONTHLY, + expected_specs=_monthly_leaf_facts("job_logs", 2026, 9, 2, level=1), + required_fragments=( + 'PARTITION OF "job_logs"', + "job_logs_2026_10", + "FOR VALUES FROM ('2026-09-01 00:00:00+00:00') " + "TO ('2026-10-01 00:00:00+00:00')", + ), + forbidden_fragments=( + "PARTITION BY", "FOR VALUES IN", "MODULUS", "DEFAULT", + ), + drop_before=DropBeforeExpectation( + cutoff=_dt(2026, 10, 1), + candidates=("job_logs_2026_09",), + ), + ), + PartitionSyntaxCase( + name="flat_range_explicit", + description=( + "Flat RANGE parent with explicit half-month bounds; explicit " + "bounds win over the default interval arithmetic." + ), + plan=_AUDIT_FLAT_EXPLICIT, + expected_specs=( + _leaf("audit_events_2026_01_01", "audit_events", "RANGE", + {"from": _dt(2026, 1, 1), "to": _dt(2026, 1, 16)}), + _leaf("audit_events_2026_01_16", "audit_events", "RANGE", + {"from": _dt(2026, 1, 16), "to": _dt(2026, 2, 1)}), + ), + required_fragments=( + 'PARTITION OF "audit_events"', + "audit_events_2026_01_16", + "FOR VALUES FROM ('2026-01-01 00:00:00+00:00') " + "TO ('2026-01-16 00:00:00+00:00')", + ), + forbidden_fragments=( + "PARTITION BY", "FOR VALUES IN", "MODULUS", "DEFAULT", + ), + drop_before=DropBeforeExpectation( + cutoff=_dt(2026, 1, 16), + candidates=("audit_events_2026_01_01",), + ), + ), + PartitionSyntaxCase( + name="flat_list_groups", + description=( + "Flat LIST parent whose policy yields one partition per value " + "group, with multi-value groups rendered as IN (...)." + ), + plan=_EVENTS_FLAT_LIST, + expected_specs=( + _leaf("events_us_ca", "events", "LIST", {"values": ("us", "ca")}), + _leaf("events_eu", "events", "LIST", {"values": ("eu",)}), + _leaf("events_apac", "events", "LIST", {"values": ("apac",)}), + ), + required_fragments=( + 'PARTITION OF "events"', + "FOR VALUES IN ('us', 'ca')", + "events_apac", + ), + forbidden_fragments=( + "PARTITION BY", "FOR VALUES FROM", "MODULUS", "DEFAULT", + ), + ), + PartitionSyntaxCase( + name="flat_hash_remainders", + description=( + "Flat HASH parent selecting a subset of remainders from a " + "modulus; buckets are named p and rendered as " + "MODULUS/REMAINDER." + ), + plan=_SESSIONS_FLAT_HASH, + expected_specs=( + _branch("sessions_p0", "sessions", "HASH", + {"modulus": 4, "remainder": 0}), + _branch("sessions_p2", "sessions", "HASH", + {"modulus": 4, "remainder": 2}), + ), + required_fragments=( + 'PARTITION OF "sessions"', + "FOR VALUES WITH (MODULUS 4, REMAINDER 0)", + "FOR VALUES WITH (MODULUS 4, REMAINDER 2)", + ), + forbidden_fragments=( + "PARTITION BY", "FOR VALUES FROM", "FOR VALUES IN", "DEFAULT", + ), + ), + PartitionSyntaxCase( + name="explicit_range_branches", + description=( + "First-level RANGE branches declared explicitly: one branch is " + "subpartitioned again, the other is a leaf." + ), + plan=_AUDIT_RANGE_BRANCHES, + expected_specs=( + _branch("audit_h1", "audit", "RANGE", + {"from": _dt(2026, 1, 1), "to": _dt(2026, 7, 1)}, + child_method="RANGE", child_columns=("created_at",)), + _leaf("audit_h1_2026_01", "audit_h1", "RANGE", + {"from": _dt(2026, 1, 1), "to": _dt(2026, 2, 1)}, level=2), + _leaf("audit_h1_2026_02", "audit_h1", "RANGE", + {"from": _dt(2026, 2, 1), "to": _dt(2026, 3, 1)}, level=2), + _branch("audit_h2", "audit", "RANGE", + {"from": _dt(2026, 7, 1), "to": _dt(2027, 1, 1)}), + ), + required_fragments=( + 'PARTITION OF "audit"', + "FOR VALUES FROM ('2026-01-01 00:00:00+00:00') " + "TO ('2026-07-01 00:00:00+00:00')", + 'PARTITION BY RANGE ("created_at")', + "audit_h1_2026_02", + ), + forbidden_fragments=("FOR VALUES IN", "MODULUS", "DEFAULT"), + drop_before=DropBeforeExpectation( + cutoff=_dt(2026, 7, 1), + candidates=("audit_h1_2026_01", "audit_h1_2026_02"), + ), + ), + PartitionSyntaxCase( + name="explicit_list_branches", + description=( + "First-level LIST branches declared explicitly: a multi-value " + "branch is subpartitioned by LIST, and a single-value branch is " + "a leaf." + ), + plan=_ORDERS_LIST_BRANCHES, + expected_specs=( + _branch("orders_closed", "orders", "LIST", + {"values": ("paid", "shipped")}, + child_method="LIST", child_columns=("region",)), + _leaf("orders_closed_eu", "orders_closed", "LIST", + {"values": ("eu",)}, level=2), + _leaf("orders_closed_us", "orders_closed", "LIST", + {"values": ("us",)}, level=2), + _branch("orders_cancelled", "orders", "LIST", + {"values": ("cancelled",)}), + ), + required_fragments=( + 'PARTITION OF "orders"', + "FOR VALUES IN ('paid', 'shipped') " + 'PARTITION BY LIST ("region")', + "orders_closed_eu", + "orders_cancelled", + ), + forbidden_fragments=("FOR VALUES FROM", "MODULUS", "DEFAULT"), + ), + PartitionSyntaxCase( + name="hash_hetero_month_quarter_day_leaf", + description=( + "HASH modulus 4 parent whose first three branches choose monthly, " + "quarterly and daily RANGE leaves; remainder 3 is a first-level " + "leaf with no PARTITION BY." + ), + plan=PostgresPartitionPlan( + parent="telemetry", + method="HASH", + columns=("tenant_bucket",), + modulus=4, + branches=( + _TELEMETRY_BRANCHES[0], + _TELEMETRY_BRANCHES[1], + _TELEMETRY_BRANCHES[2], + HashBranch(3), + ), + ), + expected_specs=( + _branch("telemetry_p0", "telemetry", "HASH", + {"modulus": 4, "remainder": 0}, + child_method="RANGE", child_columns=("created_at",)), + *_monthly_leaf_facts("telemetry_p0", 2026, 1, 12), + _branch("telemetry_p1", "telemetry", "HASH", + {"modulus": 4, "remainder": 1}, + child_method="RANGE", child_columns=("created_at",)), + *_quarterly_leaf_facts("telemetry_p1"), + _branch("telemetry_p2", "telemetry", "HASH", + {"modulus": 4, "remainder": 2}, + child_method="RANGE", child_columns=("created_at",)), + *_daily_leaf_facts("telemetry_p2", _dt(2026, 2, 1), 28), + _branch("telemetry_p3", "telemetry", "HASH", + {"modulus": 4, "remainder": 3}), + ), + required_fragments=( + 'PARTITION OF "telemetry"', + "FOR VALUES WITH (MODULUS 4, REMAINDER 3)", + 'PARTITION BY RANGE ("created_at")', + "telemetry_p0_2026_01", + "telemetry_p1_2026_q1", + "telemetry_p2_2026_02_28", + ), + forbidden_fragments=("PARTITION BY LIST", "FOR VALUES IN", "DEFAULT"), + drop_before=DropBeforeExpectation( + cutoff=_dt(2026, 2, 2), + candidates=("telemetry_p0_2026_01", "telemetry_p2_2026_02_01"), + ), + branch_leaf_counts=( + ("telemetry_p0", 12), + ("telemetry_p1", 4), + ("telemetry_p2", 28), + ("telemetry_p3", 0), + ), + ), + PartitionSyntaxCase( + name="hash_hetero_month_quarter_day_list", + description=( + "HASH modulus 4 parent whose first three branches choose RANGE " + "granularities and whose remainder 3 branch is subpartitioned by " + "LIST on a second column." + ), + plan=PostgresPartitionPlan( + parent="telemetry", + method="HASH", + columns=("tenant_bucket",), + modulus=4, + branches=( + _TELEMETRY_BRANCHES[0], + _TELEMETRY_BRANCHES[1], + _TELEMETRY_BRANCHES[2], + HashBranch( + 3, + ListPolicy( + columns=("region",), + groups=(("eu",), ("us",)), + ), + ), + ), + ), + expected_specs=( + _branch("telemetry_p0", "telemetry", "HASH", + {"modulus": 4, "remainder": 0}, + child_method="RANGE", child_columns=("created_at",)), + *_monthly_leaf_facts("telemetry_p0", 2026, 1, 12), + _branch("telemetry_p1", "telemetry", "HASH", + {"modulus": 4, "remainder": 1}, + child_method="RANGE", child_columns=("created_at",)), + *_quarterly_leaf_facts("telemetry_p1"), + _branch("telemetry_p2", "telemetry", "HASH", + {"modulus": 4, "remainder": 2}, + child_method="RANGE", child_columns=("created_at",)), + *_daily_leaf_facts("telemetry_p2", _dt(2026, 2, 1), 28), + _branch("telemetry_p3", "telemetry", "HASH", + {"modulus": 4, "remainder": 3}, + child_method="LIST", child_columns=("region",)), + _leaf("telemetry_p3_eu", "telemetry_p3", "LIST", + {"values": ("eu",)}, level=2), + _leaf("telemetry_p3_us", "telemetry_p3", "LIST", + {"values": ("us",)}, level=2), + ), + required_fragments=( + 'PARTITION OF "telemetry"', + "FOR VALUES WITH (MODULUS 4, REMAINDER 3) " + 'PARTITION BY LIST ("region")', + "telemetry_p3_eu", + "telemetry_p3_us", + "FOR VALUES IN ('eu')", + ), + forbidden_fragments=("DEFAULT",), + drop_before=DropBeforeExpectation( + cutoff=_dt(2026, 2, 2), + candidates=("telemetry_p0_2026_01", "telemetry_p2_2026_02_01"), + ), + branch_leaf_counts=( + ("telemetry_p0", 12), + ("telemetry_p1", 4), + ("telemetry_p2", 28), + ("telemetry_p3", 2), + ), + ), + PartitionSyntaxCase( + name="attach_detach", + description=( + "Statement-only case: ALTER TABLE ... ATTACH PARTITION for RANGE " + "bounds and DETACH PARTITION CONCURRENTLY on PostgreSQL 14+." + ), + clause=ClauseCase( + statements=( + ClauseStatement( + kind="attach", + build=lambda dialect: PostgresAttachPartitionStatement( + dialect=dialect, + partition_name="orders_2026_q1", + parent_table="orders", + partition_type="RANGE", + partition_values={ + "from": date(2026, 1, 1), + "to": date(2026, 4, 1), + }, + ), + expected_sql=( + 'ALTER TABLE "orders" ATTACH PARTITION ' + '"orders_2026_q1" ' + "FOR VALUES FROM ('2026-01-01') TO ('2026-04-01')" + ), + ), + ClauseStatement( + kind="detach", + build=lambda dialect: PostgresDetachPartitionStatement( + dialect=dialect, + partition_name="orders_2024", + parent_table="orders", + concurrently=True, + ), + expected_sql=( + 'ALTER TABLE "orders" DETACH PARTITION ' + '"orders_2024" CONCURRENTLY' + ), + ), + ), + concurrent_attach_rejected=True, + ), + required_fragments=( + "ATTACH PARTITION", + "DETACH PARTITION", + "CONCURRENTLY", + ), + forbidden_fragments=("CREATE TABLE", "PARTITION BY", "PARTITION OF"), + ), +] + + +@pytest.fixture(params=CASES, ids=[case.name for case in CASES]) +def partition_syntax_case(request): + """Yield one ``PartitionSyntaxCase`` per parameterized matrix row.""" + return request.param diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_partition_plan.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_partition_plan.py new file mode 100644 index 00000000..f20971b2 --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_partition_plan.py @@ -0,0 +1,1527 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_partition_plan.py +"""Unit tests for PostgreSQL partition plans and maintenance planning. + +No server is required. Plan generation, naming, DDL rendering and catalog +comparison all run against a dialect and a scripted backend, so the arithmetic +and the report logic are pinned without a live PostgreSQL scenario. +""" + +from datetime import date, datetime, timezone + +import pytest + +from rhosocial.activerecord.backend.impl.postgres.dialect import PostgresDialect +from rhosocial.activerecord.backend.partition import ( + AsyncPartitionEnsurer, + AsyncPartitionInventory, + AsyncPartitionRetention, + AsyncPartitionValidator, + PartitionEnsurer, + PartitionInfo as CorePartitionInfo, + PartitionInventory, + PartitionReport as CorePartitionReport, + PartitionRetention, + PartitionValidator, +) +from rhosocial.activerecord.backend.impl.postgres.partition import ( + AsyncPostgresPartitionMaintenance, + HashBranch, + HashPolicy, + ListBranch, + ListPolicy, + PartitionInfo, + PartitionReport, + PostgresPartitionMaintenance, + PostgresPartitionPlan, + RangeBranch, + RangePolicy, + add_interval, + default_label, + iter_range_bounds, + normalize_interval, +) + + +UTC = timezone.utc + + +def _dt(year, month, day): + return datetime(year, month, day, tzinfo=UTC) + + +def _catalog_row(name, parent, level, partition_key=None, bound=None, + is_partitioned=False, row_count=0, total_bytes=0, schema="public"): + return { + "name": name, + "parent": parent, + "level": level, + "schema_name": schema, + "partition_key": partition_key, + "bound": bound, + "row_count": row_count, + "total_bytes": total_bytes, + "is_partitioned": is_partitioned, + } + + +class _FakeTransaction: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + +class _FakeBackend: + """Scripted backend: a fixed catalog plus DDL capture.""" + + def __init__(self, rows, version=(17, 0, 0)): + self.dialect = PostgresDialect(version=version) + self.rows = list(rows) + self.executed = [] + self.transaction_count = 0 + + def fetch_all(self, sql, params=None): + return list(self.rows) + + def execute(self, sql, params=None): + self.executed.append((sql, tuple(params or ()))) + + def transaction(self): + self.transaction_count += 1 + return _FakeTransaction() + + +class _FakeAsyncTransaction: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + return False + + +class _FakeAsyncBackend(_FakeBackend): + """Async scripted backend with the same catalog and DDL capture.""" + + async def fetch_all(self, sql, params=None): + return list(self.rows) + + async def execute(self, sql, params=None): + self.executed.append((sql, tuple(params or ()))) + + def transaction(self): + self.transaction_count += 1 + return _FakeAsyncTransaction() + + +def _multilevel_plan(): + p3_bounds = [] + for month in range(1, 13): + first = _dt(2026, month, 1) + mid = _dt(2026, month, 16) + if month == 12: + nxt = _dt(2027, 1, 1) + else: + nxt = _dt(2026, month + 1, 1) + p3_bounds.append((first, mid)) + p3_bounds.append((mid, nxt)) + return PostgresPartitionPlan( + parent="logs", + method="HASH", + columns=("tenant_bucket",), + modulus=4, + branches=( + HashBranch(0, RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=12)), + HashBranch(1, RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="quarter", count=4)), + HashBranch(2, RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="year", count=2)), + HashBranch(3, RangePolicy(columns=("created_at",), bounds=tuple(p3_bounds))), + ), + ) + + +class TestIntervalArithmetic: + def test_normalize_interval_accepts_units_pairs_and_spellings(self): + assert normalize_interval("month") == (1, "month") + assert normalize_interval("15 day") == (15, "day") + assert normalize_interval((2, "quarter")) == (2, "quarter") + + def test_normalize_interval_rejects_unknown_unit(self): + with pytest.raises(ValueError, match="interval unit"): + normalize_interval("week") + + def test_add_interval_clamps_day_of_month(self): + assert add_interval(date(2026, 1, 31), "month") == date(2026, 2, 28) + assert add_interval(date(2026, 11, 30), "quarter") == date(2027, 2, 28) + + def test_add_interval_keeps_datetime_tz(self): + value = _dt(2026, 12, 31) + assert add_interval(value, "month") == _dt(2027, 1, 31) + assert add_interval(value, (15, "day")) == _dt(2027, 1, 15) + + def test_monthly_ranges_keep_the_original_day_anchor(self): + starts = [ + start + for start, _ in iter_range_bounds(date(2026, 1, 31), "month", count=4) + ] + assert starts == [ + date(2026, 1, 31), + date(2026, 2, 28), + date(2026, 3, 31), + date(2026, 4, 30), + ] + + def test_default_labels_by_granularity(self): + assert default_label(_dt(2026, 1, 1), "month") == "2026_01" + assert default_label(_dt(2026, 2, 1), "quarter") == "2026_q1" + assert default_label(_dt(2026, 1, 1), "year") == "2026" + assert default_label(_dt(2026, 1, 1), (15, "day")) == "2026_01_01" + + +class TestPolicyBounds: + def test_monthly_bounds_are_half_open(self): + policy = RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=2) + assert tuple(policy.iter_bounds()) == ( + (_dt(2026, 1, 1), _dt(2026, 2, 1)), + (_dt(2026, 2, 1), _dt(2026, 3, 1)), + ) + + def test_quarterly_bounds(self): + policy = RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="quarter", count=4) + bounds = tuple(policy.iter_bounds()) + assert bounds[0] == (_dt(2026, 1, 1), _dt(2026, 4, 1)) + assert bounds[-1] == (_dt(2026, 10, 1), _dt(2027, 1, 1)) + + def test_explicit_bounds_win_over_interval(self): + policy = RangePolicy( + columns=("created_at",), + bounds=((_dt(2026, 1, 1), _dt(2026, 1, 16)), + (_dt(2026, 1, 16), _dt(2026, 2, 1))), + ) + assert tuple(policy.iter_bounds())[1] == (_dt(2026, 1, 16), _dt(2026, 2, 1)) + + def test_policy_requires_start_or_bounds(self): + with pytest.raises(ValueError, match="requires start or bounds"): + RangePolicy(columns=("created_at",)) + + def test_generated_policy_requires_a_horizon(self): + policy = RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1)) + with pytest.raises(ValueError, match="requires count or through"): + tuple(policy.iter_bounds()) + + def test_through_expands_each_interval_to_the_same_boundary(self): + monthly = RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="month") + quarterly = RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="quarter") + yearly = RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="year") + through = _dt(2027, 1, 1) + assert len(tuple(monthly.iter_bounds(through=through))) == 12 + assert len(tuple(quarterly.iter_bounds(through=through))) == 4 + assert len(tuple(yearly.iter_bounds(through=through))) == 1 + + def test_list_and_hash_policies(self): + listed = ListPolicy(columns=("region",), groups=(("us", "ca"), ("eu",))) + assert tuple(listed.iter_groups()) == ( + (("us", "ca"), "us_ca"), + (("eu",), "eu"), + ) + hashed = HashPolicy(columns=("bucket",), modulus=4) + assert tuple(hashed.iter_remainders()) == (0, 1, 2, 3) + + +class TestPlanGeneration: + def test_heterogeneous_tree_has_four_branches_and_forty_two_leaves(self): + plan = _multilevel_plan() + specs = tuple(plan.iter_specs()) + assert len(specs) == 46 + assert [spec.name for spec in specs if spec.level == 1] == [ + "logs_p0", "logs_p1", "logs_p2", "logs_p3", + ] + leaves = [spec for spec in specs if spec.level == 2] + assert len(leaves) == 42 + assert {spec.parent for spec in leaves} == { + "logs_p0", "logs_p1", "logs_p2", "logs_p3", + } + + def test_branch_names_are_derived_not_written(self): + plan = _multilevel_plan() + names = [spec.name for spec in plan.iter_specs() if spec.level == 2] + assert "logs_p0_2026_01" in names + assert "logs_p1_2026_q1" in names + assert "logs_p2_2026" in names + assert "logs_p3_2026_01_16" in names + assert "logs_p3_2026_12_16" in names + + def test_flat_plan_keeps_tracker_shape(self): + plan = PostgresPartitionPlan( + parent="job_logs", + method="RANGE", + columns=("attempt_created_at",), + policy=RangePolicy(columns=("attempt_created_at",), start=_dt(2026, 9, 1), + interval="month", count=2), + ) + specs = tuple(plan.iter_specs()) + assert [spec.name for spec in specs] == [ + "job_logs_2026_09", + "job_logs_2026_10", + ] + assert all(spec.level == 1 and spec.is_leaf for spec in specs) + + def test_branch_name_suffix_composes_with_child_leaves(self): + plan = PostgresPartitionPlan( + parent="logs", method="HASH", columns=("bucket",), modulus=1, + branches=(HashBranch(0, RangePolicy(columns=("created_at",), + start=_dt(2026, 1, 1), + interval="month", count=1), + name="cold"),), + ) + specs = tuple(plan.iter_specs()) + assert [spec.name for spec in specs] == ["logs_cold", "logs_cold_2026_01"] + + def test_name_pattern_is_the_only_naming_hook(self): + plan = PostgresPartitionPlan( + parent="logs", + method="RANGE", + columns=("created_at",), + policy=RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=1), + name_pattern="{parent}__{suffix}", + ) + assert tuple(plan.iter_specs())[0].name == "logs__2026_01" + + def test_branches_and_policy_are_mutually_exclusive(self): + with pytest.raises(ValueError, match="mutually exclusive"): + PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1)), + branches=(RangeBranch(_dt(2026, 1, 1), _dt(2027, 1, 1)),), + ) + + def test_flat_policy_must_match_parent_method(self): + with pytest.raises(ValueError, match="policy must be a RangePolicy"): + PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=HashPolicy(columns=("bucket",), modulus=2), + ) + + def test_heterogeneous_child_policies_are_allowed(self): + plan = PostgresPartitionPlan( + parent="logs", method="HASH", columns=("bucket",), modulus=2, + branches=( + HashBranch(0, RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=1)), + HashBranch(1, ListPolicy(columns=("region",), groups=(("eu",),))), + ), + ) + assert [spec.method for spec in plan.iter_specs()][3] == "LIST" + + def test_duplicate_generated_names_are_rejected(self): + plan = PostgresPartitionPlan( + parent="logs", method="HASH", columns=("bucket",), modulus=4, + branches=(HashBranch(0), HashBranch(0)), + ) + with pytest.raises(ValueError, match="duplicate partition name"): + tuple(plan.iter_specs()) + + def test_hash_branch_requires_modulus(self): + with pytest.raises(ValueError, match="requires modulus"): + PostgresPartitionPlan( + parent="logs", method="HASH", columns=("bucket",), + branches=(HashBranch(0),), + ) + + def test_list_branch_only_for_list_parent(self): + with pytest.raises(ValueError, match="RANGE parent's branches"): + PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + branches=(ListBranch(("eu",)),), + ) + + +class TestDdlRendering: + def _maintenance(self): + return PostgresPartitionMaintenance(_FakeBackend([])) + + def test_branch_render_appends_child_partition_clause(self): + maintenance = self._maintenance() + plan = _multilevel_plan() + branch = next(plan.iter_specs()) + sql, params = maintenance._create_statement(plan, branch, if_not_exists=True) + assert sql == ( + 'CREATE TABLE IF NOT EXISTS "logs_p0" PARTITION OF "logs" ' + "FOR VALUES WITH (MODULUS 4, REMAINDER 0) " + 'PARTITION BY RANGE ("created_at")' + ) + assert params == () + + def test_leaf_render_is_range_of_utc_bounds(self): + maintenance = self._maintenance() + plan = _multilevel_plan() + leaf = tuple(plan.iter_specs())[1] + sql, params = maintenance._create_statement(plan, leaf, if_not_exists=False) + assert sql == ( + 'CREATE TABLE "logs_p0_2026_01" PARTITION OF "logs_p0" ' + "FOR VALUES FROM ('2026-01-01 00:00:00+00:00') " + "TO ('2026-02-01 00:00:00+00:00')" + ) + + def test_drop_render_never_cascades_by_default(self): + maintenance = self._maintenance() + plan = _multilevel_plan() + sql, _ = maintenance._drop_statement(plan, "logs_p0_2026_01", cascade=False) + assert sql == 'DROP TABLE IF EXISTS "logs_p0_2026_01"' + + def test_tree_query_is_a_single_parameter_expression(self): + maintenance = self._maintenance() + plan = _multilevel_plan() + sql, params = maintenance._tree_expression(plan).to_sql() + assert "WITH RECURSIVE partition_tree" in sql + assert "pg_inherits" in sql + assert "pg_get_expr(cls.relpartbound" in sql + assert sql.count("%s") == 1 + assert params == ("logs",) + + +class TestReportsWithoutServer: + def test_list_finds_missing_and_orphans(self): + plan = PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=3), + ) + rows = [ + _catalog_row("logs", None, 0, "RANGE (created_at)", None, True), + _catalog_row("logs_2026_01", "logs", 1, + bound="FOR VALUES FROM ('2026-01-01') TO ('2026-02-01')"), + _catalog_row("logs_2025_12", "logs", 1, + bound="FOR VALUES FROM ('2025-12-01') TO ('2026-01-01')"), + ] + report = PostgresPartitionMaintenance(_FakeBackend(rows)).list(plan) + assert report.missing == (("logs", "logs_2026_02"), ("logs", "logs_2026_03")) + assert report.orphans == ("logs_2025_12",) + assert report.issues == () + + def test_unbounded_plan_does_not_call_extra_partitions_orphans(self): + plan = PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1)), + ) + rows = [ + _catalog_row("logs", None, 0, "RANGE (created_at)", None, True), + _catalog_row("logs_2026_01", "logs", 1, + bound="FOR VALUES FROM ('2026-01-01') TO ('2026-02-01')"), + ] + report = PostgresPartitionMaintenance(_FakeBackend(rows)).list( + plan, through=_dt(2026, 2, 1) + ) + assert report.orphans == () + assert report.missing == () + + def test_validate_reports_gaps_and_overlaps(self): + plan = PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy( + columns=("created_at",), + bounds=( + (_dt(2026, 1, 1), _dt(2026, 2, 1)), + (_dt(2026, 3, 1), _dt(2026, 4, 1)), + (_dt(2026, 3, 15), _dt(2026, 5, 1)), + ), + ), + ) + rows = [_catalog_row("logs", None, 0, "RANGE (created_at)", None, True)] + report = PostgresPartitionMaintenance(_FakeBackend(rows)).validate(plan) + assert any(issue.startswith("gap in logs") for issue in report.issues) + assert any(issue.startswith("overlap in logs") for issue in report.issues) + + def test_validate_reports_missing_parent(self): + plan = PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy(columns=("created_at",), bounds=((_dt(2026, 1, 1), + _dt(2026, 2, 1)),)), + ) + report = PostgresPartitionMaintenance(_FakeBackend([])).validate(plan) + assert report.partitions == () + assert report.issues == ("partitioned parent logs does not exist",) + + def test_ensure_creates_one_transaction_unit_per_branch(self): + plan = PostgresPartitionPlan( + parent="logs", method="HASH", columns=("bucket",), modulus=2, + branches=(HashBranch(0), HashBranch(1)), + ) + backend = _FakeBackend([_catalog_row("logs", None, 0, "HASH (bucket)", None, True)]) + maintenance = PostgresPartitionMaintenance(backend) + report = maintenance.ensure(plan) + assert backend.transaction_count == 2 + assert [op.name for op in report.operations] == ["logs_p0", "logs_p1"] + assert all("IF NOT EXISTS" in op.sql for op in report.operations) + + def test_ensure_raises_when_parent_is_absent(self): + plan = PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy(columns=("created_at",), bounds=((_dt(2026, 1, 1), + _dt(2026, 2, 1)),)), + ) + with pytest.raises(ValueError, match="does not exist"): + PostgresPartitionMaintenance(_FakeBackend([])).ensure(plan) + + def test_drop_before_dry_run_plans_without_executing(self): + plan = PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=3), + ) + rows = [ + _catalog_row("logs", None, 0, "RANGE (created_at)", None, True), + _catalog_row("logs_2026_01", "logs", 1, + bound="FOR VALUES FROM ('2026-01-01') TO ('2026-02-01')"), + _catalog_row("logs_2026_02", "logs", 1, + bound="FOR VALUES FROM ('2026-02-01') TO ('2026-03-01')"), + _catalog_row("logs_2026_03", "logs", 1, + bound="FOR VALUES FROM ('2026-03-01') TO ('2026-04-01')"), + ] + backend = _FakeBackend(rows) + maintenance = PostgresPartitionMaintenance(backend) + report = maintenance.drop_before(plan, _dt(2026, 2, 1), dry_run=True) + assert report.dry_run is True + assert [op.name for op in report.operations] == ["logs_2026_01"] + assert backend.executed == [] + + def test_drop_before_never_drops_a_partitioned_branch(self): + plan = _multilevel_plan() + rows = [ + _catalog_row("logs", None, 0, "HASH (tenant_bucket)", None, True), + _catalog_row("logs_p0", "logs", 1, "RANGE (created_at)", + "FOR VALUES WITH (MODULUS 4, REMAINDER 0)", True), + _catalog_row("logs_p0_2026_01", "logs_p0", 2, + bound="FOR VALUES FROM ('2026-01-01') TO ('2026-02-01')"), + ] + backend = _FakeBackend(rows) + report = PostgresPartitionMaintenance(backend).drop_before( + plan, _dt(2026, 2, 1), dry_run=False + ) + dropped = [op.name for op in report.operations] + assert dropped == ["logs_p0_2026_01"] + assert "logs_p0" not in dropped + assert all('DROP TABLE IF EXISTS "' in sql for sql, _ in backend.executed) + + def test_drop_before_ignores_branches_that_start_at_or_after_cutoff(self): + plan = PostgresPartitionPlan( + parent="events", method="HASH", columns=("tenant_bucket",), modulus=4, + branches=( + HashBranch(0, RangePolicy( + columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=12)), + HashBranch(1, RangePolicy( + columns=("created_at",), start=_dt(2026, 1, 1), + interval="quarter", count=4)), + HashBranch(2, RangePolicy( + columns=("created_at",), start=_dt(2026, 2, 1), + interval="day", count=28)), + HashBranch(3), + ), + ) + rows = [ + _catalog_row("events", None, 0, "HASH (tenant_bucket)", None, True), + _catalog_row("events_p0", "events", 1, "RANGE (created_at)", + "FOR VALUES WITH (MODULUS 4, REMAINDER 0)", True), + _catalog_row("events_p1", "events", 1, "RANGE (created_at)", + "FOR VALUES WITH (MODULUS 4, REMAINDER 1)", True), + _catalog_row("events_p2", "events", 1, "RANGE (created_at)", + "FOR VALUES WITH (MODULUS 4, REMAINDER 2)", True), + _catalog_row("events_p3", "events", 1, None, + "FOR VALUES WITH (MODULUS 4, REMAINDER 3)", False), + _catalog_row("events_p0_2026_01", "events_p0", 2, + bound="FOR VALUES FROM ('2026-01-01') TO ('2026-02-01')"), + ] + backend = _FakeBackend(rows) + report = PostgresPartitionMaintenance(backend).drop_before( + plan, _dt(2026, 2, 1), dry_run=True, + ) + + assert [op.name for op in report.operations] == ["events_p0_2026_01"] + assert backend.executed == [] + + def test_drop_branch_renders_cascade(self): + plan = _multilevel_plan() + maintenance = PostgresPartitionMaintenance(_FakeBackend([])) + sql, _ = maintenance._drop_statement(plan, "logs_p0", cascade=True) + assert sql == 'DROP TABLE IF EXISTS "logs_p0" CASCADE' + + +class TestBoundaryValidation: + """The catalog's actual ``relpartbound`` text is compared with the plan.""" + + @staticmethod + def _plan(count=2): + return PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy( + columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=count, + ), + ) + + @staticmethod + def _rows(): + return [ + _catalog_row("logs", None, 0, "RANGE (created_at)", None, True), + _catalog_row( + "logs_2026_01", "logs", 1, + bound="FOR VALUES FROM ('2025-12-01 00:00:00+00') " + "TO ('2026-01-01 00:00:00+00')", + ), + _catalog_row( + "logs_2026_02", "logs", 1, + bound="FOR VALUES FROM ('2026-02-01 00:00:00+00') " + "TO ('2026-03-01 00:00:00+00')", + ), + ] + + def test_parsed_range_bounds_are_retained_in_extras(self): + text = ( + "FOR VALUES FROM ('2026-01-01 00:00:00+00') " + "TO ('2026-02-01 00:00:00+00')" + ) + rows = [_catalog_row("logs_2026_01", "logs", 1, bound=text)] + infos = PostgresPartitionMaintenance._parse_rows(rows) + + assert infos[0].bound == text + assert infos[0].extras["range_start"] == _dt(2026, 1, 1) + assert infos[0].extras["range_end"] == _dt(2026, 2, 1) + + def test_range_boundary_mismatch_is_reported(self): + report = PostgresPartitionMaintenance( + _FakeBackend(self._rows()) + ).validate(self._plan()) + + mismatches = [ + issue for issue in report.issues + if issue.startswith("boundary mismatch for logs_2026_01") + ] + assert len(mismatches) == 1 + assert "catalog 2025-12-01 00:00:00+00:00..2026-01-01 00:00:00+00:00" in ( + mismatches[0] + ) + assert "plan 2026-01-01 00:00:00+00:00..2026-02-01 00:00:00+00:00" in ( + mismatches[0] + ) + + def test_ensure_does_not_treat_a_mismatched_partition_as_present(self): + plan = self._plan(count=3) + backend = _FakeBackend(self._rows()) + report = PostgresPartitionMaintenance(backend).ensure(plan) + + assert {op.name for op in report.operations} == { + "logs_2026_01", "logs_2026_03", + } + missing_names = {name for _, name in report.missing} + assert missing_names == {"logs_2026_01", "logs_2026_03"} + assert any( + issue.startswith("boundary mismatch for logs_2026_01") + for issue in report.issues + ) + + def test_list_boundary_mismatch_is_reported(self): + plan = PostgresPartitionPlan( + parent="events", method="LIST", columns=("region",), + policy=ListPolicy(columns=("region",), groups=(("eu",), ("us",))), + ) + rows = [ + _catalog_row("events", None, 0, "LIST (region)", None, True), + _catalog_row( + "events_eu", "events", 1, + bound="FOR VALUES IN ('eu', 'ca')", + ), + _catalog_row("events_us", "events", 1, bound="FOR VALUES IN ('us')"), + ] + report = PostgresPartitionMaintenance(_FakeBackend(rows)).validate(plan) + + assert any( + issue.startswith("boundary mismatch for events_eu") + and "catalog values" in issue + for issue in report.issues + ) + assert not any("events_us" in issue for issue in report.issues) + + def test_hash_boundary_mismatch_is_reported(self): + plan = PostgresPartitionPlan( + parent="sessions", method="HASH", columns=("tenant_id",), + policy=HashPolicy( + columns=("tenant_id",), modulus=2, remainders=(0, 1), + ), + ) + rows = [ + _catalog_row("sessions", None, 0, "HASH (tenant_id)", None, True), + _catalog_row( + "sessions_p0", "sessions", 1, + bound="FOR VALUES WITH (MODULUS 2, REMAINDER 0)", + ), + _catalog_row( + "sessions_p1", "sessions", 1, + bound="FOR VALUES WITH (MODULUS 4, REMAINDER 1)", + ), + ] + report = PostgresPartitionMaintenance(_FakeBackend(rows)).validate(plan) + + assert any( + issue.startswith("boundary mismatch for sessions_p1") + and "modulus" in issue + for issue in report.issues + ) + assert not any("sessions_p0" in issue for issue in report.issues) + + def test_unparseable_bound_is_reported_not_silently_passed(self): + plan = self._plan(count=1) + rows = [ + _catalog_row("logs", None, 0, "RANGE (created_at)", None, True), + _catalog_row( + "logs_2026_01", "logs", 1, + bound="FOR VALUES FROM (MINVALUE) " + "TO ('2026-02-01 00:00:00+00')", + ), + ] + report = PostgresPartitionMaintenance(_FakeBackend(rows)).validate(plan) + + assert any( + issue.startswith("unparsed bound for logs_2026_01") + for issue in report.issues + ) + + +class TestUnboundedPlanReports: + """An unbounded plan lists the tree instead of raising.""" + + @staticmethod + def _plan(): + return PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy(columns=("created_at",), start=_dt(2026, 1, 1)), + ) + + @staticmethod + def _rows(): + return [ + _catalog_row("logs", None, 0, "RANGE (created_at)", None, True), + _catalog_row( + "logs_2026_01", "logs", 1, + bound="FOR VALUES FROM ('2026-01-01 00:00:00+00') " + "TO ('2026-02-01 00:00:00+00')", + ), + ] + + def test_list_returns_the_catalog_tree_with_a_horizon_issue(self): + plan = self._plan() + report = PostgresPartitionMaintenance(_FakeBackend(self._rows())).list(plan) + + assert len(report.partitions) == 2 + assert report.missing == () + assert report.orphans == () + assert "plan has no finite horizon" in report.issues + + def test_validate_returns_the_same_partial_report(self): + plan = self._plan() + report = PostgresPartitionMaintenance( + _FakeBackend(self._rows()) + ).validate(plan) + + assert [info.name for info in report.partitions] == [ + "logs", "logs_2026_01", + ] + assert report.missing == () + assert "plan has no finite horizon" in report.issues + + def test_ensure_rejects_unbounded_plan_but_drop_before_uses_the_cutoff(self): + plan = self._plan() + backend = _FakeBackend(self._rows()) + maintenance = PostgresPartitionMaintenance(backend) + + with pytest.raises(ValueError, match="plan has no finite horizon"): + maintenance.ensure(plan) + report = maintenance.drop_before(plan, _dt(2026, 2, 1), dry_run=True) + assert [op.name for op in report.operations] == ["logs_2026_01"] + assert backend.executed == [] + + +class TestConcurrentDetach: + """``drop_before`` can render detach-then-drop on PostgreSQL 14+.""" + + @staticmethod + def _plan(count=2): + return PostgresPartitionPlan( + parent="logs", method="RANGE", columns=("created_at",), + policy=RangePolicy( + columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=count, + ), + ) + + @staticmethod + def _rows(): + return [ + _catalog_row("logs", None, 0, "RANGE (created_at)", None, True), + _catalog_row( + "logs_2026_01", "logs", 1, + bound="FOR VALUES FROM ('2026-01-01 00:00:00+00') " + "TO ('2026-02-01 00:00:00+00')", + ), + _catalog_row( + "logs_2026_02", "logs", 1, + bound="FOR VALUES FROM ('2026-02-01 00:00:00+00') " + "TO ('2026-03-01 00:00:00+00')", + ), + ] + + def test_dry_run_renders_detach_then_drop_without_executing(self): + backend = _FakeBackend(self._rows()) + report = PostgresPartitionMaintenance(backend).drop_before( + self._plan(), _dt(2026, 2, 1), dry_run=True, + concurrent_detach=True, + ) + + assert report.dry_run is True + assert [(op.action, op.name) for op in report.operations] == [ + ("detach", "logs_2026_01"), ("drop", "logs_2026_01"), + ] + assert report.operations[0].sql == ( + 'ALTER TABLE "logs" DETACH PARTITION "logs_2026_01" CONCURRENTLY' + ) + assert report.operations[1].sql == 'DROP TABLE IF EXISTS "logs_2026_01"' + assert backend.executed == [] + + def test_pg14_executes_detach_then_drop_without_a_transaction(self): + backend = _FakeBackend(self._rows(), version=(17, 0, 0)) + PostgresPartitionMaintenance(backend).drop_before( + self._plan(), _dt(2026, 2, 1), dry_run=False, + concurrent_detach=True, + ) + + assert backend.executed == [ + ( + 'ALTER TABLE "logs" DETACH PARTITION "logs_2026_01" CONCURRENTLY', + (), + ), + ('DROP TABLE IF EXISTS "logs_2026_01"', ()), + ] + assert backend.transaction_count == 0 + + def test_pg13_falls_back_to_the_plain_drop(self): + backend = _FakeBackend(self._rows(), version=(13, 0, 0)) + assert backend.dialect.supports_concurrent_detach() is False + report = PostgresPartitionMaintenance(backend).drop_before( + self._plan(), _dt(2026, 2, 1), dry_run=True, + concurrent_detach=True, + ) + + assert [(op.action, op.name) for op in report.operations] == [ + ("drop", "logs_2026_01"), + ] + assert "DETACH PARTITION" not in report.operations[0].sql + + def test_pg13_fallback_still_uses_one_transaction_per_unit(self): + backend = _FakeBackend(self._rows(), version=(13, 0, 0)) + PostgresPartitionMaintenance(backend).drop_before( + self._plan(), _dt(2026, 2, 1), dry_run=False, + concurrent_detach=True, + ) + + assert backend.executed == [('DROP TABLE IF EXISTS "logs_2026_01"', ())] + assert backend.transaction_count == 1 + + def test_concurrent_detach_false_keeps_the_plain_drop_on_pg17(self): + backend = _FakeBackend(self._rows(), version=(17, 0, 0)) + report = PostgresPartitionMaintenance(backend).drop_before( + self._plan(), _dt(2026, 2, 1), dry_run=True, + ) + + assert [op.action for op in report.operations] == ["drop"] + + +class TestAsyncBoundaryAndHorizon: + """Async parity for boundary validation, unbounded list and detach.""" + + @pytest.mark.asyncio + async def test_boundary_mismatch_is_reported(self): + plan = TestBoundaryValidation._plan() + backend = _FakeAsyncBackend(TestBoundaryValidation._rows()) + report = await AsyncPostgresPartitionMaintenance(backend).validate(plan) + + assert any( + issue.startswith("boundary mismatch for logs_2026_01") + for issue in report.issues + ) + + @pytest.mark.asyncio + async def test_unbounded_list_returns_the_catalog_tree(self): + plan = TestUnboundedPlanReports._plan() + backend = _FakeAsyncBackend(TestUnboundedPlanReports._rows()) + report = await AsyncPostgresPartitionMaintenance(backend).list(plan) + + assert len(report.partitions) == 2 + assert report.missing == () + assert "plan has no finite horizon" in report.issues + + @pytest.mark.asyncio + async def test_unbounded_ensure_rejects_but_drop_before_uses_the_cutoff(self): + plan = TestUnboundedPlanReports._plan() + backend = _FakeAsyncBackend(TestUnboundedPlanReports._rows()) + maintenance = AsyncPostgresPartitionMaintenance(backend) + + with pytest.raises(ValueError, match="plan has no finite horizon"): + await maintenance.ensure(plan) + report = await maintenance.drop_before(plan, _dt(2026, 2, 1), dry_run=True) + assert [op.name for op in report.operations] == ["logs_2026_01"] + + @pytest.mark.asyncio + async def test_concurrent_detach_dry_run_renders_both_statements(self): + plan = TestConcurrentDetach._plan() + backend = _FakeAsyncBackend(TestConcurrentDetach._rows()) + report = await AsyncPostgresPartitionMaintenance(backend).drop_before( + plan, _dt(2026, 2, 1), dry_run=True, concurrent_detach=True, + ) + + assert [(op.action, op.name) for op in report.operations] == [ + ("detach", "logs_2026_01"), ("drop", "logs_2026_01"), + ] + assert backend.executed == [] + + @pytest.mark.asyncio + async def test_async_pg13_falls_back_to_the_plain_drop(self): + plan = TestConcurrentDetach._plan() + backend = _FakeAsyncBackend(TestConcurrentDetach._rows(), version=(13, 0, 0)) + report = await AsyncPostgresPartitionMaintenance(backend).drop_before( + plan, _dt(2026, 2, 1), dry_run=True, concurrent_detach=True, + ) + + assert [op.action for op in report.operations] == ["drop"] + + +class TestPortableMaintenanceContract: + """The PostgreSQL executor speaks the core contract's shape and verbs.""" + + def test_reports_are_the_core_value_objects(self): + assert PartitionInfo is CorePartitionInfo + assert PartitionReport is CorePartitionReport + + def test_catalog_bytes_map_to_the_core_size_field(self): + rows = [_catalog_row("logs", None, 0, total_bytes=8192)] + infos = PostgresPartitionMaintenance._parse_rows(rows) + + assert infos[0].size_bytes == 8192 + + def test_the_sync_executor_offers_every_canonical_verb(self): + executor = PostgresPartitionMaintenance(_FakeBackend([])) + + assert isinstance(executor, PartitionInventory) + assert isinstance(executor, PartitionEnsurer) + assert isinstance(executor, PartitionRetention) + assert isinstance(executor, PartitionValidator) + + def test_the_async_executor_offers_every_canonical_verb(self): + executor = AsyncPostgresPartitionMaintenance(_FakeBackend([])) + + assert isinstance(executor, AsyncPartitionInventory) + assert isinstance(executor, AsyncPartitionEnsurer) + assert isinstance(executor, AsyncPartitionRetention) + assert isinstance(executor, AsyncPartitionValidator) + + +def _batch_plan(parent, count=2, interval="month"): + return PostgresPartitionPlan( + parent=parent, + method="RANGE", + columns=("created_at",), + policy=RangePolicy( + columns=("created_at",), + start=_dt(2026, 1, 1), + interval=interval, + count=count, + ), + ) + + +def _range_bound(start, end): + return "FOR VALUES FROM ('%s') TO ('%s')" % ( + start.isoformat(), + end.isoformat(), + ) + + +def _root_row(parent): + return _catalog_row(parent, None, 0, "RANGE (created_at)", None, True) + + +def _planned_row(parent, label, start, end): + return _catalog_row( + "%s_%s" % (parent, label), + parent, + 1, + "RANGE (created_at)", + _range_bound(start, end), + ) + + +def _rows_for_root(rows, root): + """Return the rows belonging to *root* plus their descendants.""" + names = {root} + changed = True + while changed: + changed = False + for row in rows: + if row["parent"] in names and row["name"] not in names: + names.add(row["name"]) + changed = True + return [row for row in rows if row["name"] in names] + + +def _apply_catalog_change(backend, sql): + """Apply a rendered CREATE/DROP to the scripted catalogue.""" + if sql.startswith("CREATE TABLE") and "PARTITION OF" in sql: + name = sql.split('"')[1] + parent = sql.split("PARTITION OF ")[1].split('"')[1] + bound = None + if "FOR VALUES" in sql: + bound = sql[sql.index("FOR VALUES"):] + if " PARTITION BY " in bound: + bound = bound.split(" PARTITION BY ")[0].strip() + partition_key = None + if bound and bound.startswith("FOR VALUES FROM"): + partition_key = "RANGE (created_at)" + elif bound and bound.startswith("FOR VALUES IN"): + partition_key = "LIST (region)" + elif bound and bound.startswith("FOR VALUES WITH"): + partition_key = "HASH (bucket)" + backend.rows.append( + _catalog_row( + name, + parent, + 1, + partition_key, + bound, + "PARTITION BY" in sql, + ) + ) + return + if sql.startswith("DROP TABLE"): + name = sql.split('"')[1] + backend.rows = [row for row in backend.rows if row["name"] != name] + + +class _SnapshotTransaction: + def __init__(self, backend): + self.backend = backend + + def __enter__(self): + self.snapshot = list(self.backend.rows) + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is not None: + self.backend.rows = self.snapshot + return False + + +class _AsyncSnapshotTransaction: + def __init__(self, backend): + self.backend = backend + + async def __aenter__(self): + self.snapshot = list(self.backend.rows) + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + if exc_type is not None: + self.backend.rows = self.snapshot + return False + + +class _MutableCatalogBackend(_FakeBackend): + """Scripted backend that applies rendered DDL to its catalogue.""" + + def __init__(self, rows, version=(17, 0, 0), fail_on=None): + super().__init__(rows, version) + self.fail_on = fail_on + + def fetch_all(self, sql, params=None): + root = params[0] if params else None + return _rows_for_root(self.rows, root) + + def execute(self, sql, params=None): + if self.fail_on is not None and self.fail_on in sql: + raise RuntimeError("scripted failure: %s" % self.fail_on) + super().execute(sql, params) + _apply_catalog_change(self, sql) + + def transaction(self): + self.transaction_count += 1 + return _SnapshotTransaction(self) + + +class _MutableCatalogAsyncBackend(_FakeAsyncBackend): + """Async scripted backend that applies rendered DDL to its catalogue.""" + + def __init__(self, rows, version=(17, 0, 0), fail_on=None): + super().__init__(rows, version) + self.fail_on = fail_on + + async def fetch_all(self, sql, params=None): + root = params[0] if params else None + return _rows_for_root(self.rows, root) + + async def execute(self, sql, params=None): + if self.fail_on is not None and self.fail_on in sql: + raise RuntimeError("scripted failure: %s" % self.fail_on) + self.executed.append((sql, tuple(params or ()))) + _apply_catalog_change(self, sql) + + def transaction(self): + self.transaction_count += 1 + return _AsyncSnapshotTransaction(self) + + +class TestPartitionBatchEnsure: + def test_independent_batch_creates_every_plan_in_one_transaction(self): + plan_a = _batch_plan("job_logs", count=2) + plan_b = _batch_plan("step_logs", count=2) + backend = _MutableCatalogBackend([_root_row("job_logs"), _root_row("step_logs")]) + + batch = PostgresPartitionMaintenance(backend).ensure_many( + plans=[plan_a, plan_b] + ) + + assert backend.transaction_count == 1 + assert set(batch.reports) == {"job_logs", "step_logs"} + assert batch.pairs == () + assert batch.differences == () + assert all(report.missing == () for report in batch.reports.values()) + assert len(backend.executed) == 4 + + def test_a_later_plan_failure_rolls_back_the_whole_batch(self): + plan_a = _batch_plan("job_logs", count=2) + plan_b = _batch_plan("step_logs", count=2) + backend = _MutableCatalogBackend( + [_root_row("job_logs"), _root_row("step_logs")], + fail_on="step_logs_2026_01", + ) + + with pytest.raises(RuntimeError, match="scripted failure"): + PostgresPartitionMaintenance(backend).ensure_many( + plans=[plan_a, plan_b] + ) + + assert backend.transaction_count == 1 + assert [row["name"] for row in backend.rows] == ["job_logs", "step_logs"] + assert any('"job_logs_2026_01"' in sql for sql, _ in backend.executed) + assert all( + '"step_logs_2026_01"' not in sql for sql, _ in backend.executed + ) + + def test_paired_ensure_fills_both_sides_and_leaves_no_drift(self): + plan_a = _batch_plan("job_logs", count=2) + plan_b = _batch_plan("step_logs", count=2) + backend = _MutableCatalogBackend([_root_row("job_logs"), _root_row("step_logs")]) + + batch = PostgresPartitionMaintenance(backend).ensure_many( + pairs=[("log_tables", plan_a, plan_b)] + ) + + assert batch.pairs == ("log_tables",) + assert batch.differences == () + assert batch.issues == () + assert all(report.missing == () for report in batch.reports.values()) + + +class TestPartitionBatchList: + def test_paired_list_reports_left_only_and_right_only_buckets(self): + left_plan = _batch_plan("job_logs", count=3) + right_plan = _batch_plan("step_logs", count=3) + rows = [ + _root_row("job_logs"), + _planned_row("job_logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + _planned_row("job_logs", "2026_02", _dt(2026, 2, 1), _dt(2026, 3, 1)), + _root_row("step_logs"), + _planned_row("step_logs", "2026_02", _dt(2026, 2, 1), _dt(2026, 3, 1)), + _planned_row("step_logs", "2026_03", _dt(2026, 3, 1), _dt(2026, 4, 1)), + ] + batch = PostgresPartitionMaintenance(_MutableCatalogBackend(rows)).list_many( + pairs=[("log_tables", left_plan, right_plan)] + ) + + assert [ + (diff.present_in, diff.left_partition, diff.right_partition) + for diff in batch.differences + ] == [ + ("left", "job_logs_2026_01", None), + ("right", None, "step_logs_2026_03"), + ] + assert batch.issues == () + + def test_two_side_absence_stays_in_each_report_missing(self): + left_plan = _batch_plan("job_logs", count=3) + right_plan = _batch_plan("step_logs", count=3) + rows = [ + _root_row("job_logs"), + _planned_row("job_logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + _root_row("step_logs"), + _planned_row("step_logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + ] + batch = PostgresPartitionMaintenance(_MutableCatalogBackend(rows)).list_many( + pairs=[("log_tables", left_plan, right_plan)] + ) + + assert batch.differences == () + assert {name for _, name in batch.reports["job_logs"].missing} == { + "job_logs_2026_02", "job_logs_2026_03", + } + assert {name for _, name in batch.reports["step_logs"].missing} == { + "step_logs_2026_02", "step_logs_2026_03", + } + + def test_boundary_text_disagreement_is_an_issue_not_a_difference(self): + left_plan = _batch_plan("job_logs", count=1) + right_plan = _batch_plan("step_logs", count=1) + rows = [ + _root_row("job_logs"), + _catalog_row( + "job_logs_2026_01", "job_logs", 1, "RANGE (created_at)", + "FOR VALUES FROM ('2026-01-01 00:00:00+00') " + "TO ('2026-02-01 00:00:00+00')", + ), + _root_row("step_logs"), + _catalog_row( + "step_logs_2026_01", "step_logs", 1, "RANGE (created_at)", + "FOR VALUES FROM ('2026-01-01 00:00:00+00:00') " + "TO ('2026-02-01 00:00:00+00:00')", + ), + ] + batch = PostgresPartitionMaintenance(_MutableCatalogBackend(rows)).list_many( + pairs=[("log_tables", left_plan, right_plan)] + ) + + assert batch.differences == () + assert len(batch.issues) == 1 + assert batch.issues[0].startswith("boundary mismatch for log_tables/RANGE|") + + def test_incompatible_pair_is_an_issue_with_no_differences(self): + left_plan = _batch_plan("job_logs", count=2, interval="month") + right_plan = _batch_plan("step_logs", count=2, interval="quarter") + rows = [_root_row("job_logs"), _root_row("step_logs")] + batch = PostgresPartitionMaintenance(_MutableCatalogBackend(rows)).list_many( + pairs=[("log_tables", left_plan, right_plan)] + ) + + assert batch.differences == () + assert batch.issues == ("pair log_tables plans are not comparable",) + + +class TestPartitionBatchDrop: + def test_dry_run_reports_before_state_and_executes_nothing(self): + plan = _batch_plan("logs", count=3) + backend = _MutableCatalogBackend([ + _root_row("logs"), + _planned_row("logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + ]) + + batch = PostgresPartitionMaintenance(backend).drop_before_many( + plans=[plan], + cutoff=_dt(2026, 2, 1), + dry_run=True, + through=_dt(2026, 4, 1), + ) + + assert backend.executed == [] + assert backend.transaction_count == 0 + report = batch.reports["logs"] + assert report.dry_run is True + assert [(op.action, op.name) for op in report.operations] == [ + ("drop", "logs_2026_01"), + ] + assert {name for _, name in report.missing} == { + "logs_2026_02", "logs_2026_03", + } + assert any(info.name == "logs_2026_01" for info in report.partitions) + + def test_real_run_uses_one_transaction_and_reports_post_state(self): + plan = _batch_plan("logs", count=3) + backend = _MutableCatalogBackend([ + _root_row("logs"), + _planned_row("logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + ]) + + batch = PostgresPartitionMaintenance(backend).drop_before_many( + plans=[plan], + cutoff=_dt(2026, 2, 1), + dry_run=False, + through=_dt(2026, 4, 1), + ) + + assert backend.transaction_count == 1 + assert [op.name for op in batch.reports["logs"].operations] == [ + "logs_2026_01", + ] + assert "logs_2026_01" not in { + info.name for info in batch.reports["logs"].partitions + } + assert {name for _, name in batch.reports["logs"].missing} == { + "logs_2026_02", "logs_2026_03", + } + + def test_concurrent_detach_with_several_plans_keeps_one_transaction(self): + plan_a = _batch_plan("job_logs", count=2) + plan_b = _batch_plan("step_logs", count=2) + backend = _MutableCatalogBackend([_root_row("job_logs"), _root_row("step_logs")]) + + batch = PostgresPartitionMaintenance(backend).drop_before_many( + plans=[plan_a, plan_b], + cutoff=_dt(2026, 2, 1), + dry_run=False, + concurrent_detach=True, + ) + + assert any( + "concurrent detach requires a single plan" in issue + for issue in batch.issues + ) + assert backend.transaction_count == 1 + assert all( + op.action == "drop" + for report in batch.reports.values() + for op in report.operations + ) + + def test_concurrent_detach_with_one_plan_skips_the_transaction(self): + plan = _batch_plan("logs", count=3) + backend = _MutableCatalogBackend([ + _root_row("logs"), + _planned_row("logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + ]) + + batch = PostgresPartitionMaintenance(backend).drop_before_many( + plans=[plan], + cutoff=_dt(2026, 2, 1), + dry_run=False, + concurrent_detach=True, + ) + + assert backend.transaction_count == 0 + assert [(op.action, op.name) for op in batch.reports["logs"].operations] == [ + ("detach", "logs_2026_01"), + ("drop", "logs_2026_01"), + ] + + +class TestAsyncPartitionBatchEnsure: + @pytest.mark.asyncio + async def test_independent_batch_creates_every_plan_in_one_transaction(self): + plan_a = _batch_plan("job_logs", count=2) + plan_b = _batch_plan("step_logs", count=2) + backend = _MutableCatalogAsyncBackend([ + _root_row("job_logs"), _root_row("step_logs"), + ]) + + batch = await AsyncPostgresPartitionMaintenance(backend).ensure_many( + plans=[plan_a, plan_b] + ) + + assert backend.transaction_count == 1 + assert set(batch.reports) == {"job_logs", "step_logs"} + assert batch.differences == () + assert all(report.missing == () for report in batch.reports.values()) + assert len(backend.executed) == 4 + + @pytest.mark.asyncio + async def test_a_later_plan_failure_rolls_back_the_whole_batch(self): + plan_a = _batch_plan("job_logs", count=2) + plan_b = _batch_plan("step_logs", count=2) + backend = _MutableCatalogAsyncBackend( + [_root_row("job_logs"), _root_row("step_logs")], + fail_on="step_logs_2026_01", + ) + + with pytest.raises(RuntimeError, match="scripted failure"): + await AsyncPostgresPartitionMaintenance(backend).ensure_many( + plans=[plan_a, plan_b] + ) + + assert backend.transaction_count == 1 + assert [row["name"] for row in backend.rows] == ["job_logs", "step_logs"] + + @pytest.mark.asyncio + async def test_paired_ensure_fills_both_sides_and_leaves_no_drift(self): + plan_a = _batch_plan("job_logs", count=2) + plan_b = _batch_plan("step_logs", count=2) + backend = _MutableCatalogAsyncBackend([ + _root_row("job_logs"), _root_row("step_logs"), + ]) + + batch = await AsyncPostgresPartitionMaintenance(backend).ensure_many( + pairs=[("log_tables", plan_a, plan_b)] + ) + + assert batch.pairs == ("log_tables",) + assert batch.differences == () + assert all(report.missing == () for report in batch.reports.values()) + + +class TestAsyncPartitionBatchList: + @pytest.mark.asyncio + async def test_paired_list_reports_left_only_and_right_only_buckets(self): + left_plan = _batch_plan("job_logs", count=3) + right_plan = _batch_plan("step_logs", count=3) + rows = [ + _root_row("job_logs"), + _planned_row("job_logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + _planned_row("job_logs", "2026_02", _dt(2026, 2, 1), _dt(2026, 3, 1)), + _root_row("step_logs"), + _planned_row("step_logs", "2026_02", _dt(2026, 2, 1), _dt(2026, 3, 1)), + _planned_row("step_logs", "2026_03", _dt(2026, 3, 1), _dt(2026, 4, 1)), + ] + batch = await AsyncPostgresPartitionMaintenance( + _MutableCatalogAsyncBackend(rows) + ).list_many(pairs=[("log_tables", left_plan, right_plan)]) + + assert [ + (diff.present_in, diff.left_partition, diff.right_partition) + for diff in batch.differences + ] == [ + ("left", "job_logs_2026_01", None), + ("right", None, "step_logs_2026_03"), + ] + + @pytest.mark.asyncio + async def test_two_side_absence_stays_in_each_report_missing(self): + left_plan = _batch_plan("job_logs", count=3) + right_plan = _batch_plan("step_logs", count=3) + rows = [ + _root_row("job_logs"), + _planned_row("job_logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + _root_row("step_logs"), + _planned_row("step_logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + ] + batch = await AsyncPostgresPartitionMaintenance( + _MutableCatalogAsyncBackend(rows) + ).list_many(pairs=[("log_tables", left_plan, right_plan)]) + + assert batch.differences == () + assert {name for _, name in batch.reports["job_logs"].missing} == { + "job_logs_2026_02", "job_logs_2026_03", + } + + @pytest.mark.asyncio + async def test_boundary_text_disagreement_is_an_issue_not_a_difference(self): + left_plan = _batch_plan("job_logs", count=1) + right_plan = _batch_plan("step_logs", count=1) + rows = [ + _root_row("job_logs"), + _catalog_row( + "job_logs_2026_01", "job_logs", 1, "RANGE (created_at)", + "FOR VALUES FROM ('2026-01-01 00:00:00+00') " + "TO ('2026-02-01 00:00:00+00')", + ), + _root_row("step_logs"), + _catalog_row( + "step_logs_2026_01", "step_logs", 1, "RANGE (created_at)", + "FOR VALUES FROM ('2026-01-01 00:00:00+00:00') " + "TO ('2026-02-01 00:00:00+00:00')", + ), + ] + batch = await AsyncPostgresPartitionMaintenance( + _MutableCatalogAsyncBackend(rows) + ).list_many(pairs=[("log_tables", left_plan, right_plan)]) + + assert batch.differences == () + assert len(batch.issues) == 1 + assert batch.issues[0].startswith("boundary mismatch for log_tables/RANGE|") + + @pytest.mark.asyncio + async def test_incompatible_pair_is_an_issue_with_no_differences(self): + left_plan = _batch_plan("job_logs", count=2, interval="month") + right_plan = _batch_plan("step_logs", count=2, interval="quarter") + rows = [_root_row("job_logs"), _root_row("step_logs")] + batch = await AsyncPostgresPartitionMaintenance( + _MutableCatalogAsyncBackend(rows) + ).list_many(pairs=[("log_tables", left_plan, right_plan)]) + + assert batch.differences == () + assert batch.issues == ("pair log_tables plans are not comparable",) + + +class TestAsyncPartitionBatchDrop: + @pytest.mark.asyncio + async def test_dry_run_reports_before_state_and_executes_nothing(self): + plan = _batch_plan("logs", count=3) + backend = _MutableCatalogAsyncBackend([ + _root_row("logs"), + _planned_row("logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + ]) + + batch = await AsyncPostgresPartitionMaintenance(backend).drop_before_many( + plans=[plan], + cutoff=_dt(2026, 2, 1), + dry_run=True, + through=_dt(2026, 4, 1), + ) + + assert backend.executed == [] + assert backend.transaction_count == 0 + report = batch.reports["logs"] + assert report.dry_run is True + assert [(op.action, op.name) for op in report.operations] == [ + ("drop", "logs_2026_01"), + ] + assert any(info.name == "logs_2026_01" for info in report.partitions) + + @pytest.mark.asyncio + async def test_real_run_uses_one_transaction_and_reports_post_state(self): + plan = _batch_plan("logs", count=3) + backend = _MutableCatalogAsyncBackend([ + _root_row("logs"), + _planned_row("logs", "2026_01", _dt(2026, 1, 1), _dt(2026, 2, 1)), + ]) + + batch = await AsyncPostgresPartitionMaintenance(backend).drop_before_many( + plans=[plan], + cutoff=_dt(2026, 2, 1), + dry_run=False, + through=_dt(2026, 4, 1), + ) + + assert backend.transaction_count == 1 + assert [op.name for op in batch.reports["logs"].operations] == [ + "logs_2026_01", + ] + assert "logs_2026_01" not in { + info.name for info in batch.reports["logs"].partitions + } + + @pytest.mark.asyncio + async def test_concurrent_detach_with_several_plans_keeps_one_transaction(self): + plan_a = _batch_plan("job_logs", count=2) + plan_b = _batch_plan("step_logs", count=2) + backend = _MutableCatalogAsyncBackend([ + _root_row("job_logs"), _root_row("step_logs"), + ]) + + batch = await AsyncPostgresPartitionMaintenance(backend).drop_before_many( + plans=[plan_a, plan_b], + cutoff=_dt(2026, 2, 1), + dry_run=False, + concurrent_detach=True, + ) + + assert any( + "concurrent detach requires a single plan" in issue + for issue in batch.issues + ) + assert backend.transaction_count == 1 + assert all( + op.action == "drop" + for report in batch.reports.values() + for op in report.operations + ) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_partition_syntax_matrix.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_partition_syntax_matrix.py new file mode 100644 index 00000000..1ab7ab7b --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_partition_syntax_matrix.py @@ -0,0 +1,297 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/postgres/test_partition_syntax_matrix.py +"""Offline matrix for PostgreSQL partition syntax families. + +Each test consumes ``partition_syntax_case`` from +``partition_syntax_fixtures.py``. The matrix proves plan generation, spec +shape, DDL rendering, retention selection and protocol conformance without a +PostgreSQL server: a dialect plus a scripted catalog is enough. + +The only statement-only row is ``attach_detach``; every other row is +plan-driven and is checked against its declared expected specs rather than +against names written inside the test. +""" + +from datetime import datetime + +import pytest + +from rhosocial.activerecord.backend.dialect.exceptions import UnsupportedFeatureError +from rhosocial.activerecord.backend.impl.postgres.dialect import PostgresDialect +from rhosocial.activerecord.backend.impl.postgres.expression.ddl.partition import ( + PostgresAttachPartitionStatement, +) +from rhosocial.activerecord.backend.impl.postgres.partition import ( + PostgresPartitionMaintenance, +) +from rhosocial.activerecord.backend.impl.postgres.protocols.ddl.partition import ( + PostgresPartitionSupport, +) +from rhosocial.activerecord.backend.partition import ( + PartitionEnsurer, + PartitionInventory, + PartitionRetention, + PartitionValidator, +) +from partition_syntax_fixtures import ( # noqa: F401 (fixture re-export) + PartitionSyntaxCase, + partition_syntax_case as _matrix_case, +) + + +@pytest.fixture(name="partition_syntax_case") +def _partition_syntax_case_fixture(_matrix_case): # noqa: F811 + """Expose the parameterized matrix fixture under its public name.""" + return _matrix_case + + +_METHOD_FRAGMENTS = { + "RANGE": "FOR VALUES FROM", + "LIST": "FOR VALUES IN", + "HASH": "FOR VALUES WITH", +} + + +class _FakeTransaction: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + +class _FakeBackend: + """Scripted backend: a fixed catalog plus DDL capture.""" + + def __init__(self, rows): + self.dialect = PostgresDialect(version=(17, 0, 0)) + self.rows = list(rows) + self.executed = [] + + def fetch_all(self, sql, params=None): + return list(self.rows) + + def execute(self, sql, params=None): + self.executed.append((sql, tuple(params or ()))) + + def transaction(self): + return _FakeTransaction() + + +def _format_bound_value(value): + if isinstance(value, datetime): + return value.isoformat(sep=" ") + return str(value) + + +def _bound_text(bounds): + if "modulus" in bounds: + return "FOR VALUES WITH (MODULUS %d, REMAINDER %d)" % ( + bounds["modulus"], bounds["remainder"], + ) + if "values" in bounds: + values = ", ".join("'%s'" % value for value in bounds["values"]) + return "FOR VALUES IN (%s)" % values + return "FOR VALUES FROM ('%s') TO ('%s')" % ( + _format_bound_value(bounds["from"]), + _format_bound_value(bounds["to"]), + ) + + +def _catalog_rows(case: PartitionSyntaxCase): + """Build the catalog a live server would return for the fixture facts.""" + plan = case.plan + if plan is None: + return [] + root_key = "%s (%s)" % (plan.method, ", ".join(plan.columns)) + rows = [{ + "name": plan.parent, + "parent": None, + "level": 0, + "schema_name": "public", + "partition_key": root_key, + "bound": None, + "row_count": 0, + "total_bytes": 0, + "is_partitioned": True, + }] + for fact in case.expected_specs: + partition_key = None + if fact.child_method is not None: + partition_key = "%s (%s)" % ( + fact.child_method, ", ".join(fact.child_columns), + ) + rows.append({ + "name": fact.name, + "parent": fact.parent, + "level": fact.level, + "schema_name": "public", + "partition_key": partition_key, + "bound": _bound_text(fact.bounds), + "row_count": 0, + "total_bytes": 0, + "is_partitioned": fact.child_method is not None, + }) + return rows + + +def _plan_ddl(case: PartitionSyntaxCase, if_not_exists=True): + """Render the plan's actual specs, keyed by partition name.""" + maintenance = PostgresPartitionMaintenance(_FakeBackend([])) + return { + spec.name: maintenance._create_statement(case.plan, spec, if_not_exists)[0] + for spec in case.plan.iter_specs() + } + + +def _clause_sql(case: PartitionSyntaxCase): + dialect = PostgresDialect(version=(17, 0, 0)) + return [ + statement.build(dialect).to_sql() + for statement in case.clause.statements + ] + + +def _rendered_sql(case: PartitionSyntaxCase): + if case.plan is not None: + return tuple(_plan_ddl(case).values()) + return tuple(sql for sql, _ in _clause_sql(case)) + + +def test_plan_specs_match_fixture_facts(partition_syntax_case): + case = partition_syntax_case + if case.plan is None: + pytest.skip("statement-only case carries no plan specs") + + actual = tuple(case.plan.iter_specs()) + assert len(actual) == len(case.expected_specs) + assert [spec.name for spec in actual] == [ + fact.name for fact in case.expected_specs + ] + + for spec, fact in zip(actual, case.expected_specs): + assert spec.name == fact.name + assert spec.parent == fact.parent + assert spec.level == fact.level + assert spec.is_leaf is fact.is_leaf + assert spec.method == fact.method + assert spec.bounds == fact.bounds + assert spec.child_method == fact.child_method + assert spec.child_columns == fact.child_columns + + +def test_branch_leaf_counts_and_shape(partition_syntax_case): + case = partition_syntax_case + if not case.branch_leaf_counts: + pytest.skip("flat or statement-only case has no branch leaf counts") + + actual = tuple(case.plan.iter_specs()) + for branch_name, expected_count in case.branch_leaf_counts: + branch = next(spec for spec in actual if spec.name == branch_name) + leaves = [ + spec for spec in actual + if spec.parent == branch_name and spec.level == 2 + ] + assert branch.level == 1 + assert len(leaves) == expected_count + if expected_count: + assert branch.is_leaf is False + assert branch.child_method is not None + else: + assert branch.is_leaf is True + assert branch.child_method is None + + +def test_leaf_and_subpartitioned_branch_ddl(partition_syntax_case): + case = partition_syntax_case + if case.plan is None: + pytest.skip("statement-only case has no plan DDL") + + ddl = _plan_ddl(case) + for spec in case.plan.iter_specs(): + sql = ddl[spec.name] + assert _METHOD_FRAGMENTS[spec.method] in sql + if spec.child_method is None: + assert "PARTITION BY" not in sql + else: + assert "PARTITION BY %s" % spec.child_method in sql + for column in spec.child_columns: + assert column in sql + + +def test_ddl_required_and_forbidden_fragments(partition_syntax_case): + case = partition_syntax_case + rendered = "\n".join(_rendered_sql(case)) + for fragment in case.required_fragments: + assert fragment in rendered + for fragment in case.forbidden_fragments: + assert fragment not in rendered + + +def test_drop_before_candidate_selection(partition_syntax_case): + case = partition_syntax_case + if case.drop_before is None: + pytest.skip("case has no range leaves or no retention expectation") + + backend = _FakeBackend(_catalog_rows(case)) + maintenance = PostgresPartitionMaintenance(backend) + report = maintenance.drop_before( + case.plan, case.drop_before.cutoff, dry_run=True, + ) + + assert report.dry_run is True + assert [op.name for op in report.operations] == list(case.drop_before.candidates) + assert backend.executed == [] + + facts = {fact.name: fact for fact in case.expected_specs} + for operation in report.operations: + assert facts[operation.name].is_leaf is True + + +def test_clause_statements_render(partition_syntax_case): + case = partition_syntax_case + if case.clause is None: + pytest.skip("plan-driven case has no standalone clause statements") + + for statement, (sql, params) in zip( + case.clause.statements, _clause_sql(case), + ): + assert sql == statement.expected_sql + assert params == () + + if case.clause.concurrent_attach_rejected: + dialect = PostgresDialect(version=(17, 0, 0)) + statement = PostgresAttachPartitionStatement( + dialect=dialect, + partition_name="orders_2026_q1", + parent_table="orders", + partition_type="RANGE", + partition_values={"from": "2026-01-01", "to": "2026-04-01"}, + concurrently=True, + ) + with pytest.raises(UnsupportedFeatureError): + statement.to_sql() + assert dialect.supports_concurrent_attach() is False + + +def test_protocol_conformance(partition_syntax_case): + case = partition_syntax_case + dialect = PostgresDialect(version=(17, 0, 0)) + assert isinstance(dialect, PostgresPartitionSupport) + assert dialect.supports_table_partitioning() + assert dialect.supports_range_table_partitioning() + assert dialect.supports_list_table_partitioning() + assert dialect.supports_hash_table_partitioning() + assert dialect.supports_default_partition() + assert dialect.supports_attach_partition() + assert dialect.supports_detach_partition() + + expected_detach = dialect.supports_concurrent_detach() + assert expected_detach is True + assert dialect.supports_concurrent_attach() is False + assert dialect.supports_reorganize_partition() is False + + executor = PostgresPartitionMaintenance(_FakeBackend(_catalog_rows(case))) + assert isinstance(executor, PartitionInventory) + assert isinstance(executor, PartitionEnsurer) + assert isinstance(executor, PartitionRetention) + assert isinstance(executor, PartitionValidator) diff --git a/tests/rhosocial/activerecord_postgres_test/feature/backend/test_partition_multilevel.py b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_partition_multilevel.py new file mode 100644 index 00000000..72525dfe --- /dev/null +++ b/tests/rhosocial/activerecord_postgres_test/feature/backend/test_partition_multilevel.py @@ -0,0 +1,518 @@ +# tests/rhosocial/activerecord_postgres_test/feature/backend/test_partition_multilevel.py +"""Live PostgreSQL tests for multi-level, heterogeneous partitioning. + +The tree under test is PostgreSQL's own capability, not a lowest-common- +denominator shape: a HASH parent whose four branches each pick a different +RANGE granularity. + + logs (HASH tenant_bucket, modulus 4) + |-- p0 RANGE created_at, monthly, 12 leaves + |-- p1 RANGE created_at, quarterly, 4 leaves + |-- p2 RANGE created_at, yearly, 2 leaves + `-- p3 RANGE created_at, explicit half-month spans, 24 leaves + +These tests require a live PostgreSQL 11+ server. They prove: + +* each level's DDL is accepted by the server (not only rendered); +* a row is routed by HASH to a branch and by RANGE to the correct leaf + (``tableoid``), with ``[start, end)`` boundaries and explicit UTC bounds; +* the parent primary key must contain every level's partition key, and a + server rejection is pinned as a negative case; +* maintenance is idempotent, reports missing/orphan partitions, drops only + leaves, evaluates the cutoff per branch, and removes a whole branch only + through the explicit ``drop_branch`` call. + +Unit coverage for the same plan without a server lives in +``postgres/test_partition_plan.py``. +""" + +import re +from datetime import datetime, timezone + +import pytest +import pytest_asyncio + +from rhosocial.activerecord.backend.errors import DatabaseError +from rhosocial.activerecord.backend.expression import Column, Literal +from rhosocial.activerecord.backend.expression.objects import Table +from rhosocial.activerecord.backend.expression.statements import ( + ColumnConstraint, + ColumnDefinition, + ColumnNotNullConstraint, + CreateTableStatement, + DropTableStatement, + InsertStatement, + PartitionClause, + PartitionStrategy, + TableConstraint, + TablePrimaryKeyConstraint, + ValuesSource, +) +from rhosocial.activerecord.backend.expression.types import ( + BigIntType, + TextType, + TimestampTzType, +) +from rhosocial.activerecord.backend.impl.postgres.expression.ddl.partition import ( + PostgresCreatePartitionStatement, +) +from rhosocial.activerecord.backend.impl.postgres.partition import ( + AsyncPostgresPartitionMaintenance, + HashBranch, + PostgresPartitionMaintenance, + PostgresPartitionPlan, + RangePolicy, +) + +TABLE = "ar_partition_multi_logs" +BAD_TABLE = "ar_partition_multi_badlogs" +ORPHAN = "ar_partition_multi_logs_p0_2019_01" + +UTC = timezone.utc + +P0 = "ar_partition_multi_logs_p0" +P1 = "ar_partition_multi_logs_p1" +P2 = "ar_partition_multi_logs_p2" +P3 = "ar_partition_multi_logs_p3" + + +def _dt(year, month, day, hour=0, minute=0, second=0, microsecond=0): + return datetime(year, month, day, hour, minute, second, microsecond, tzinfo=UTC) + + +def _half_month_bounds(): + """Two spans per month: [1st, 16th) and [16th, next 1st).""" + bounds = [] + for month in range(1, 13): + first = _dt(2026, month, 1) + middle = _dt(2026, month, 16) + following = _dt(2027, 1, 1) if month == 12 else _dt(2026, month + 1, 1) + bounds.append((first, middle)) + bounds.append((middle, following)) + return tuple(bounds) + + +def _plan(): + return PostgresPartitionPlan( + parent=TABLE, + method="HASH", + columns=("tenant_bucket",), + modulus=4, + branches=( + HashBranch(0, RangePolicy( + columns=("created_at",), start=_dt(2026, 1, 1), + interval="month", count=12)), + HashBranch(1, RangePolicy( + columns=("created_at",), start=_dt(2026, 1, 1), + interval="quarter", count=4)), + HashBranch(2, RangePolicy( + columns=("created_at",), start=_dt(2026, 1, 1), + interval="year", count=2)), + HashBranch(3, RangePolicy( + columns=("created_at",), bounds=_half_month_bounds())), + ), + ) + + +def _parent_columns(dialect): + return [ + ColumnDefinition( + dialect, "id", BigIntType(dialect=dialect), + constraints=[ColumnConstraint(dialect, branch=ColumnNotNullConstraint(dialect))], + ), + ColumnDefinition( + dialect, "tenant_bucket", BigIntType(dialect=dialect), + constraints=[ColumnConstraint(dialect, branch=ColumnNotNullConstraint(dialect))], + ), + ColumnDefinition( + dialect, "created_at", TimestampTzType(dialect=dialect), + constraints=[ColumnConstraint(dialect, branch=ColumnNotNullConstraint(dialect))], + ), + ColumnDefinition(dialect, "payload", TextType(dialect=dialect)), + ] + + +def _create_parent(backend, table, primary_key): + dialect = backend.dialect + statement = CreateTableStatement( + dialect=dialect, + table=Table(dialect, table), + columns=_parent_columns(dialect), + table_constraints=[ + TableConstraint( + dialect, branch=TablePrimaryKeyConstraint(dialect, primary_key) + ) + ], + partition=PartitionClause( + dialect, PartitionStrategy.HASH, [Column(dialect, "tenant_bucket")] + ), + ) + backend.execute(*statement.to_sql()) + + +def _drop_tables(backend, *names): + for name in names: + statement = DropTableStatement( + dialect=backend.dialect, + table=Table(backend.dialect, name), + if_exists=True, + cascade=True, + ) + backend.execute(*statement.to_sql()) + + +def _insert_rows(backend, rows): + dialect = backend.dialect + statement = InsertStatement( + dialect=dialect, + into=Table(dialect, TABLE), + columns=["id", "tenant_bucket", "created_at", "payload"], + source=ValuesSource( + dialect, + [[Literal(dialect, value) for value in row] for row in rows], + ), + ) + backend.execute(*statement.to_sql()) + + +def _require_multilevel(backend): + dialect = backend.dialect + if not dialect.supports_subpartitioning(): + pytest.skip("PostgreSQL server does not support subpartitioning") + if not dialect.supports_hash_partitioning(): + pytest.skip("PostgreSQL server does not support HASH partitioning") + + +def _leaf_counts(report): + counts = {P0: 0, P1: 0, P2: 0, P3: 0} + for info in report.partitions: + if info.level == 2: + counts[info.parent] += 1 + return counts + + +def _probe_routing(backend): + """Insert probe rows and return ``{branch_index: bucket}``. + + HASH partitioning uses ``hashint4``, not ``bucket % modulus``, so the + mapping is read from the server rather than assumed. Every probe row must + also satisfy ``satisfies_hash_partition`` for the branch it landed in. + """ + _insert_rows( + backend, + [ + [100 + bucket, bucket, _dt(2026, 1, 15, 10, 0), "probe-%d" % bucket] + for bucket in range(64) + ], + ) + check_sql = ( + "SELECT tenant_bucket, tableoid::regclass::text AS leaf, " + "satisfies_hash_partition(" + "'public.%s'::regclass, 4, " + "(regexp_match(tableoid::regclass::text, '_p(\\d+)_'))[1]::int, " + "tenant_bucket) AS ok " + "FROM %s WHERE id >= 100 AND id < 200" % (TABLE, TABLE) + ) + rows = backend.fetch_all(check_sql) + by_branch = {} + for row in rows: + assert row["ok"] is True, row + branch = int(re.search(r"_p(\d+)_", row["leaf"]).group(1)) + by_branch[branch] = row["tenant_bucket"] + assert sorted(by_branch) == [0, 1, 2, 3] + return by_branch + + +def _assert_tree(report): + root = next(info for info in report.partitions if info.level == 0) + assert root.name == TABLE + assert root.method == "HASH" + branch_methods = { + info.name: info.method for info in report.partitions if info.level == 1 + } + assert branch_methods == {P0: "RANGE", P1: "RANGE", P2: "RANGE", P3: "RANGE"} + assert _leaf_counts(report) == {P0: 12, P1: 4, P2: 2, P3: 24} + assert report.missing == () + assert report.issues == () + + +def _assert_primary_key(backend, table, expected_columns): + rows = backend.fetch_all( + "SELECT pg_get_constraintdef(c.oid) AS definition " + "FROM pg_constraint c JOIN pg_class t ON t.oid = c.conrelid " + "WHERE t.relname = %s AND c.contype = 'p'", + (table,), + ) + assert len(rows) == 1 + definition = rows[0]["definition"] + for column in expected_columns: + assert column in definition + + +class TestPostgresPartitionMultilevel: + """Synchronous live tests for a heterogeneous two-level tree.""" + + @pytest.fixture + def multilevel_plan(self, postgres_backend_single): + backend = postgres_backend_single + _require_multilevel(backend) + _drop_tables(backend, TABLE, ORPHAN) + _create_parent(backend, TABLE, ["id", "tenant_bucket", "created_at"]) + yield _plan() + _drop_tables(backend, TABLE, ORPHAN) + + def test_creates_heterogeneous_nested_tree(self, postgres_backend_single, multilevel_plan): + """The server accepts every branch's own RANGE granularity.""" + backend = postgres_backend_single + maintenance = PostgresPartitionMaintenance(backend) + report = maintenance.ensure(multilevel_plan) + + assert len(report.operations) == 46 + branch_sql = [op.sql for op in report.operations if op.level == 1] + assert all("PARTITION BY RANGE" in sql for sql in branch_sql) + assert all("FOR VALUES WITH (MODULUS 4" in sql for sql in branch_sql) + leaf_sql = [op.sql for op in report.operations if op.level == 2] + assert all("FOR VALUES FROM" in sql for sql in leaf_sql) + + _assert_tree(report) + _assert_primary_key( + backend, TABLE, ["id", "tenant_bucket", "created_at"] + ) + + again = maintenance.ensure(multilevel_plan) + assert again.operations == () + _assert_tree(again) + + def test_routes_rows_by_hash_branch_then_range_leaf( + self, postgres_backend_single, multilevel_plan + ): + """A row's branch follows the hash key and its leaf follows the bound.""" + backend = postgres_backend_single + PostgresPartitionMaintenance(backend).ensure(multilevel_plan) + buckets = _probe_routing(backend) + + boundary = [ + # branch, created_at, leaf suffix + (0, _dt(2026, 1, 31, 23, 59, 59, 999999), "2026_01"), + (0, _dt(2026, 2, 1, 0, 0), "2026_02"), + (1, _dt(2026, 3, 31, 23, 59, 59, 999999), "2026_q1"), + (1, _dt(2026, 4, 1, 0, 0), "2026_q2"), + (2, _dt(2026, 12, 31, 23, 59, 59, 999999), "2026"), + (2, _dt(2027, 1, 1, 0, 0), "2027"), + (3, _dt(2026, 1, 15, 23, 59, 59, 999999), "2026_01_01"), + (3, _dt(2026, 1, 16, 0, 0), "2026_01_16"), + ] + expected = {} + rows = [] + for index, (branch, when, suffix) in enumerate(boundary): + row_id = 200 + index + expected[row_id] = "%s_p%d_%s" % (TABLE, branch, suffix) + rows.append([row_id, buckets[branch], when, "boundary-%d" % index]) + _insert_rows(backend, rows) + + landed = backend.fetch_all( + "SELECT id, tableoid::regclass::text AS leaf FROM %s " + "WHERE id >= 200 AND id < 300 ORDER BY id" % TABLE + ) + assert {row["id"]: row["leaf"] for row in landed} == expected + + def test_parent_primary_key_must_include_all_partition_keys( + self, postgres_backend_single, multilevel_plan + ): + """A PK without the second-level key makes the server reject the tree.""" + backend = postgres_backend_single + _drop_tables(backend, BAD_TABLE) + try: + _create_parent(backend, BAD_TABLE, ["id", "tenant_bucket"]) + statement = PostgresCreatePartitionStatement( + dialect=backend.dialect, + partition_name=BAD_TABLE + "_p0", + parent_table=BAD_TABLE, + partition_type="HASH", + partition_values={"modulus": 4, "remainder": 0}, + partition_clause=PartitionClause( + backend.dialect, + PartitionStrategy.RANGE, + [Column(backend.dialect, "created_at")], + ), + ) + with pytest.raises( + DatabaseError, match="must include all partitioning columns" + ): + backend.execute(*statement.to_sql()) + finally: + _drop_tables(backend, BAD_TABLE) + + def test_list_reports_missing_leaf_and_orphan_branch( + self, postgres_backend_single, multilevel_plan + ): + """list finds a dropped expected leaf and an extra catalog partition.""" + backend = postgres_backend_single + maintenance = PostgresPartitionMaintenance(backend) + maintenance.ensure(multilevel_plan) + + missing = P1 + "_2026_q2" + backend.execute( + *DropTableStatement( + dialect=backend.dialect, + table=Table(backend.dialect, missing), + if_exists=True, + ).to_sql() + ) + backend.execute( + *PostgresCreatePartitionStatement( + dialect=backend.dialect, + partition_name=ORPHAN, + parent_table=P0, + partition_type="RANGE", + partition_values={ + "from": _dt(2019, 1, 1), + "to": _dt(2019, 2, 1), + }, + ).to_sql() + ) + + report = maintenance.list(multilevel_plan) + assert report.missing == ((P1, missing),) + assert ORPHAN in report.orphans + assert report.issues == () + + validate = maintenance.validate(multilevel_plan) + assert validate.missing == ((P1, missing),) + row = backend.fetch_one( + "SELECT to_regclass(%s) AS present", (P1 + "_2026_q2",) + ) + assert row["present"] is None + + def test_drop_before_evaluates_each_branch_policy( + self, postgres_backend_single, multilevel_plan + ): + """One cutoff removes different counts per branch; dry run is silent.""" + backend = postgres_backend_single + maintenance = PostgresPartitionMaintenance(backend) + maintenance.ensure(multilevel_plan) + cutoff = _dt(2026, 2, 1) + + planned = maintenance.drop_before(multilevel_plan, cutoff, dry_run=True) + assert planned.dry_run is True + assert [op.name for op in planned.operations] == [ + P0 + "_2026_01", + P3 + "_2026_01_01", + P3 + "_2026_01_16", + ] + assert P0 not in {op.name for op in planned.operations} + + dropped = maintenance.drop_before(multilevel_plan, cutoff, dry_run=False) + assert [op.name for op in dropped.operations] == [ + op.name for op in planned.operations + ] + assert dropped.missing == () + remaining = {info.name for info in dropped.partitions} + assert P0 + "_2026_02" in remaining + assert P1 + "_2026_q1" in remaining + assert P2 + "_2026" in remaining + + noop = maintenance.drop_before(multilevel_plan, cutoff, dry_run=False) + assert noop.operations == () + + def test_drop_branch_removes_one_branch_and_ensure_rebuilds_it( + self, postgres_backend_single, multilevel_plan + ): + """drop_branch cascades its leaves; siblings and ensure are unaffected.""" + backend = postgres_backend_single + maintenance = PostgresPartitionMaintenance(backend) + maintenance.ensure(multilevel_plan) + before = maintenance.list(multilevel_plan) + + report = maintenance.drop_branch(multilevel_plan, multilevel_plan.branches[0]) + assert [op.name for op in report.operations] == [P0] + assert report.missing == () + remaining = {info.name for info in report.partitions} + assert P0 not in remaining + assert P0 + "_2026_01" not in remaining + assert P1 in remaining + assert len([info for info in report.partitions if info.parent == P1]) == 4 + + rebuilt = maintenance.ensure(multilevel_plan) + assert len(rebuilt.operations) == 13 + assert {op.name for op in rebuilt.operations} == { + info.name for info in before.partitions if info.level > 0 and + (info.name == P0 or info.parent == P0) + } + _assert_tree(rebuilt) + + +class TestAsyncPostgresPartitionMultilevel: + """Asynchronous live tests; method names mirror the sync class.""" + + @pytest_asyncio.fixture + async def async_multilevel_plan(self, async_postgres_backend_single): + backend = async_postgres_backend_single + _require_multilevel(backend) + for name in (TABLE, ORPHAN): + statement = DropTableStatement( + dialect=backend.dialect, + table=Table(backend.dialect, name), + if_exists=True, + cascade=True, + ) + await backend.execute(*statement.to_sql()) + dialect = backend.dialect + statement = CreateTableStatement( + dialect=dialect, + table=Table(dialect, TABLE), + columns=_parent_columns(dialect), + table_constraints=[ + TableConstraint( + dialect, + branch=TablePrimaryKeyConstraint( + dialect, ["id", "tenant_bucket", "created_at"] + ), + ) + ], + partition=PartitionClause( + dialect, PartitionStrategy.HASH, [Column(dialect, "tenant_bucket")] + ), + ) + await backend.execute(*statement.to_sql()) + yield _plan() + for name in (TABLE, ORPHAN): + drop = DropTableStatement( + dialect=backend.dialect, + table=Table(backend.dialect, name), + if_exists=True, + cascade=True, + ) + await backend.execute(*drop.to_sql()) + + @pytest.mark.asyncio + async def test_creates_heterogeneous_nested_tree( + self, async_postgres_backend_single, async_multilevel_plan + ): + """The async executor builds the same heterogeneous tree.""" + backend = async_postgres_backend_single + maintenance = AsyncPostgresPartitionMaintenance(backend) + report = await maintenance.ensure(async_multilevel_plan) + assert len(report.operations) == 46 + _assert_tree(report) + + again = await maintenance.ensure(async_multilevel_plan) + assert again.operations == () + + @pytest.mark.asyncio + async def test_drop_before_evaluates_each_branch_policy( + self, async_postgres_backend_single, async_multilevel_plan + ): + """The async executor drops the same per-branch leaf set.""" + backend = async_postgres_backend_single + maintenance = AsyncPostgresPartitionMaintenance(backend) + await maintenance.ensure(async_multilevel_plan) + planned = await maintenance.drop_before( + async_multilevel_plan, _dt(2026, 2, 1), dry_run=True + ) + assert [op.name for op in planned.operations] == [ + P0 + "_2026_01", + P3 + "_2026_01_01", + P3 + "_2026_01_16", + ] + assert planned.dry_run is True